@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
import unicodedata
|
||||
|
||||
source = Path('C:/Mac/Home/Documents/eBAU/cartografia_moz.xlsx')
|
||||
checksum = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
@@ -12,15 +13,24 @@ if checksum != '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff
|
||||
raise ValueError('Authorised workbook changed')
|
||||
ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
||||
country = str(uuid.uuid5(uuid.NAMESPACE_URL, 'investmoz:workbook:MZ'))
|
||||
nodes = {(): dict(id=country, parent_id=None, name='Moçambique', location_type='country', code='508')}
|
||||
nodes = {(): dict(id=country, parent_id=None, name='Moçambique', location_type='country', code='508', source_code='508')}
|
||||
definitions = {}
|
||||
conflicts = set()
|
||||
rejected = []
|
||||
ignored = 0
|
||||
read = 0
|
||||
duplicates = 0
|
||||
with zipfile.ZipFile(source) as archive:
|
||||
strings = [''.join(si.itertext()) for si in ET.fromstring(archive.read('xl/sharedStrings.xml')).findall('s:si', ns)]
|
||||
sheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
|
||||
headers = []
|
||||
for cell in sheet.findall('s:sheetData/s:row', ns)[0]:
|
||||
value = cell.findtext('s:v', '', ns)
|
||||
headers.append(strings[int(value)] if cell.attrib.get('t') == 's' else value)
|
||||
if len(headers) < 8 or 'designa' not in unicodedata.normalize('NFKD', headers[7]).casefold():
|
||||
raise ValueError('Unexpected workbook headers; inspect before importing')
|
||||
for row in sheet.findall('s:sheetData/s:row', ns)[1:]:
|
||||
read += 1
|
||||
cells = {}
|
||||
invalid = False
|
||||
for cell in row:
|
||||
@@ -32,7 +42,7 @@ with zipfile.ZipFile(source) as archive:
|
||||
value = ''.join(cell.find('s:is', ns).itertext())
|
||||
if cell.find('s:f', ns) is not None or cell.attrib.get('t') == 'e':
|
||||
invalid = True
|
||||
cells[column] = ' '.join(value.split())
|
||||
cells[column] = unicodedata.normalize('NFC', ' '.join(value.split()))
|
||||
if cells.get('F') or cells.get('G'):
|
||||
ignored += 1
|
||||
continue
|
||||
@@ -47,11 +57,14 @@ with zipfile.ZipFile(source) as archive:
|
||||
if key in conflicts:
|
||||
rejected.append({'row': row.attrib['r'], 'reason': 'conflicting_designation'})
|
||||
continue
|
||||
if key in definitions and definitions[key]['name'] != name:
|
||||
if key in definitions and definitions[key]['name'].casefold() != name.casefold():
|
||||
rejected.extend([{'row': definitions[key]['row'], 'reason': 'conflicting_designation'}, {'row': row.attrib['r'], 'reason': 'conflicting_designation'}])
|
||||
conflicts.add(key)
|
||||
del definitions[key]
|
||||
continue
|
||||
if key in definitions:
|
||||
duplicates += 1
|
||||
continue
|
||||
definitions[key] = {'name': name, 'row': row.attrib['r']}
|
||||
for key, definition in sorted(definitions.items(), key=lambda item: (len(item[0]), item[0])):
|
||||
if key[:-1] not in nodes:
|
||||
@@ -60,10 +73,10 @@ for key, definition in sorted(definitions.items(), key=lambda item: (len(item[0]
|
||||
parent = nodes[key[:-1]]['id']
|
||||
kind = ['province', 'district', 'admin_post', 'locality'][len(key)-1]
|
||||
identifier = str(uuid.uuid5(uuid.NAMESPACE_URL, parent + ':' + kind + ':' + key[-1].casefold()))
|
||||
nodes[key] = dict(id=identifier, parent_id=parent, name=definition['name'], location_type=kind, code='XLSX-' + identifier)
|
||||
nodes[key] = dict(id=identifier, parent_id=parent, name=definition['name'], location_type=kind, code='XLSX-' + identifier, source_code=key[-1])
|
||||
output = Path('storage/app/private/territory-workbook-candidates.json')
|
||||
previous = output.with_name('territory-workbook-candidates-before-designation-fix.json')
|
||||
if output.exists() and not previous.exists():
|
||||
previous.write_bytes(output.read_bytes())
|
||||
output.write_text(json.dumps(dict(checksum=checksum, mapping_version=2, nodes=list(nodes.values()), rejected_rows=rejected, ignored_sub_locality_rows=ignored), ensure_ascii=False), encoding='utf-8')
|
||||
output.write_text(json.dumps(dict(checksum=checksum, mapping_version=3, headers=headers, read=read, duplicates=duplicates, nodes=list(nodes.values()), rejected_rows=rejected, ignored_sub_locality_rows=ignored), ensure_ascii=False), encoding='utf-8')
|
||||
print(json.dumps(dict(counts={kind: sum(n['location_type'] == kind for n in nodes.values()) for kind in ['country','province','district','admin_post','locality']}, rejected_rows=len(rejected), ignored_sub_locality_rows=ignored)))
|
||||
|
||||
Reference in New Issue
Block a user