83 lines
4.4 KiB
Python
83 lines
4.4 KiB
Python
"""B:G contain codes. Designacao names only the deepest populated level."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import uuid
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
import unicodedata
|
|
|
|
source = Path('C:/Mac/Home/Documents/eBAU/cartografia_moz.xlsx')
|
|
checksum = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
if checksum != '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff':
|
|
raise ValueError('Authorised workbook changed')
|
|
ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
|
country = str(uuid.uuid5(uuid.NAMESPACE_URL, 'investmoz:workbook:MZ'))
|
|
nodes = {(): dict(id=country, parent_id=None, name='Moçambique', location_type='country', code='508', source_code='508')}
|
|
definitions = {}
|
|
conflicts = set()
|
|
rejected = []
|
|
ignored = 0
|
|
read = 0
|
|
duplicates = 0
|
|
with zipfile.ZipFile(source) as archive:
|
|
strings = [''.join(si.itertext()) for si in ET.fromstring(archive.read('xl/sharedStrings.xml')).findall('s:si', ns)]
|
|
sheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
|
|
headers = []
|
|
for cell in sheet.findall('s:sheetData/s:row', ns)[0]:
|
|
value = cell.findtext('s:v', '', ns)
|
|
headers.append(strings[int(value)] if cell.attrib.get('t') == 's' else value)
|
|
if len(headers) < 8 or 'designa' not in unicodedata.normalize('NFKD', headers[7]).casefold():
|
|
raise ValueError('Unexpected workbook headers; inspect before importing')
|
|
for row in sheet.findall('s:sheetData/s:row', ns)[1:]:
|
|
read += 1
|
|
cells = {}
|
|
invalid = False
|
|
for cell in row:
|
|
column = ''.join(c for c in cell.attrib['r'] if c.isalpha())
|
|
value = cell.findtext('s:v', '', ns)
|
|
if cell.attrib.get('t') == 's':
|
|
value = strings[int(value)]
|
|
elif cell.attrib.get('t') == 'inlineStr':
|
|
value = ''.join(cell.find('s:is', ns).itertext())
|
|
if cell.find('s:f', ns) is not None or cell.attrib.get('t') == 'e':
|
|
invalid = True
|
|
cells[column] = unicodedata.normalize('NFC', ' '.join(value.split()))
|
|
if cells.get('F') or cells.get('G'):
|
|
ignored += 1
|
|
continue
|
|
segments = [cells.get(column, '') for column in 'BCDE']
|
|
while segments and not segments[-1]:
|
|
segments.pop()
|
|
if invalid or not segments or '' in segments or not cells.get('H'):
|
|
rejected.append({'row': row.attrib['r'], 'reason': 'missing_code_path_or_designation'})
|
|
continue
|
|
key = tuple(segments)
|
|
name = cells['H']
|
|
if key in conflicts:
|
|
rejected.append({'row': row.attrib['r'], 'reason': 'conflicting_designation'})
|
|
continue
|
|
if key in definitions and definitions[key]['name'].casefold() != name.casefold():
|
|
rejected.extend([{'row': definitions[key]['row'], 'reason': 'conflicting_designation'}, {'row': row.attrib['r'], 'reason': 'conflicting_designation'}])
|
|
conflicts.add(key)
|
|
del definitions[key]
|
|
continue
|
|
if key in definitions:
|
|
duplicates += 1
|
|
continue
|
|
definitions[key] = {'name': name, 'row': row.attrib['r']}
|
|
for key, definition in sorted(definitions.items(), key=lambda item: (len(item[0]), item[0])):
|
|
if key[:-1] not in nodes:
|
|
rejected.append({'row': definition['row'], 'reason': 'missing_parent_definition'})
|
|
continue
|
|
parent = nodes[key[:-1]]['id']
|
|
kind = ['province', 'district', 'admin_post', 'locality'][len(key)-1]
|
|
identifier = str(uuid.uuid5(uuid.NAMESPACE_URL, parent + ':' + kind + ':' + key[-1].casefold()))
|
|
nodes[key] = dict(id=identifier, parent_id=parent, name=definition['name'], location_type=kind, code='XLSX-' + identifier, source_code=key[-1])
|
|
output = Path('storage/app/private/territory-workbook-candidates.json')
|
|
previous = output.with_name('territory-workbook-candidates-before-designation-fix.json')
|
|
if output.exists() and not previous.exists():
|
|
previous.write_bytes(output.read_bytes())
|
|
output.write_text(json.dumps(dict(checksum=checksum, mapping_version=3, headers=headers, read=read, duplicates=duplicates, nodes=list(nodes.values()), rejected_rows=rejected, ignored_sub_locality_rows=ignored), ensure_ascii=False), encoding='utf-8')
|
|
print(json.dumps(dict(counts={kind: sum(n['location_type'] == kind for n in nodes.values()) for kind in ['country','province','district','admin_post','locality']}, rejected_rows=len(rejected), ignored_sub_locality_rows=ignored)))
|