feat(proactive): integrate private supply observations and demonstrate Gurue comparison in QA

This commit is contained in:
2026-10-28 17:19:55 +02:00
parent 150bef4b89
commit 2f58b7d438
19 changed files with 1104 additions and 19 deletions
+172
View File
@@ -0,0 +1,172 @@
"""Read-only XLSX ingestion into a private, indexed analysis snapshot. No publication.
Every source row remains addressable. Matching identifiers suggest the same entity;
they never imply a current trading status, contract, or an exhaustive census.
"""
import argparse
import hashlib
import json
from pathlib import Path
import sqlite3
import unicodedata
import xml.etree.ElementTree as ET
import zipfile
NS = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
SOURCES = {
'companies.xlsx': ('1ba4c4114b16b407e7746712d00d4f681246bbbc1b42351e96a31f7129acad30', 'EMPRESAS_LIST 2014-2022.xlsx', '2014–2022; data por registo quando disponível'),
'licensed.xlsx': ('1b5087b464e39e88c42def39655c2391aaaf684b4c832d6a77ff274b98889f00', 'Empresas Licenciadas.xlsx', 'Data por registo; cobertura temporal global não determinada'),
'tourism.xlsx': ('f02e086ed4973ef17234aa6cc54b98fbb7a6952ffb16ec4dc4f7d0c72af6400c', 'CADASTRO DE EMPREENDIMENTOS TURISTICOS 3, 4 E 5 ESTRELAS 30.10.24 18,00h.xlsx', '30/10/2024 (nome do exemplar); situação actual não confirmada'),
'cae.xlsx': ('83b4f1200988e99834c2ae3915346fbf237d128308554b67b5a080d4956bf506', 'CAE eBAU.xlsx', 'Versão não indicada'),
}
def norm(v):
return ''.join(c for c in unicodedata.normalize('NFKD', str(v)).casefold() if not unicodedata.combining(c)).strip()
def digest(v):
return hashlib.sha256(v.encode('utf-8')).hexdigest()
def rows(path):
# Shared strings are spooled to a temporary on-disk SQLite lookup, not a huge XML tree.
with zipfile.ZipFile(path) as z, sqlite3.connect('') as strings:
strings.execute('create table strings(id integer primary key,value text)')
if 'xl/sharedStrings.xml' in z.namelist():
with z.open('xl/sharedStrings.xml') as f:
i = 0
for _, e in ET.iterparse(f, events=('end',)):
if e.tag == NS+'si':
strings.execute('insert into strings values(?,?)', (i, ''.join(t.text or '' for t in e.iter(NS+'t'))))
i += 1
e.clear()
book = ET.fromstring(z.read('xl/workbook.xml'))
rels = {r.attrib['Id']: r.attrib['Target'] for r in ET.fromstring(z.read('xl/_rels/workbook.xml.rels'))}
for sheet in book.find(NS+'sheets'):
target = rels[sheet.attrib['{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id']]
target = target.lstrip('/') if target.startswith('/') else 'xl/'+target
with z.open(target) as f:
for _, e in ET.iterparse(f, events=('end',)):
if e.tag != NS+'row':
continue
result = {}
for c in e:
col = ''.join(ch for ch in c.attrib.get('r','') if ch.isalpha())
v = c.findtext(NS+'v', '')
if c.attrib.get('t') == 's':
hit = strings.execute('select value from strings where id=?', (int(v),)).fetchone()
v = hit[0] if hit else ''
elif c.attrib.get('t') == 'inlineStr':
v = ''.join(t.text or '' for t in c.iter(NS+'t'))
result[col] = ' '.join(v.split())
yield sheet.attrib['name'], int(e.attrib['r']), result
e.clear()
def main():
p = argparse.ArgumentParser()
p.add_argument('--sources', required=True)
p.add_argument('--output', required=True)
p.add_argument('--only-tourism', action='store_true', help='Rebuild only tourism in an existing private preview')
a = p.parse_args()
output = Path(a.output)
output.mkdir(parents=True, exist_ok=True)
manifest = {'version':1, 'access':'internal', 'sources':[], 'publication':False,
'limitations':'Registos administrativos históricos; cobertura incompleta, actividade actual, capacidade, procura e compromissos não confirmados. Ausência de registo significa informação insuficiente.'}
for alias, (expected, name, period) in SOURCES.items():
path = Path(a.sources)/alias
actual = hashlib.file_digest(path.open('rb'), 'sha256').hexdigest()
if actual != expected:
raise ValueError('Original changed: '+alias)
manifest['sources'].append({'alias':alias,'name':name,'sha256':actual,'bytes':path.stat().st_size,'period':period})
db = sqlite3.connect(output/'supply-preview.sqlite')
db.executescript('''create table if not exists records(id text primary key, source_hash text, sheet text, row_number integer, entity_key text, name text, province text, district text, cae text, activity text, sector text, source_date text, fingerprint text, duplicate_of text, identity_conflict integer default 0);
create index if not exists supply_place on records(province,district);
create index if not exists supply_entity on records(entity_key);
create index if not exists supply_fingerprint on records(fingerprint);''')
labels = {}
hierarchy = {}
for _, row, c in rows(Path(a.sources)/'cae.xlsx'):
if row > 1 and c.get('B') and c.get('F'):
labels[c['B']] = c['F']
hierarchy[c['A']] = c
# Supplied CAE hierarchy labels retained. Sector is the top-level CAE section,
# not an invented correspondence to the portal's twelve navigation sectors.
sector_by_code = {}
for node in hierarchy.values():
ancestor=node
visited=set()
while ancestor.get('D')!='1' and ancestor.get('C') in hierarchy and ancestor['A'] not in visited:
visited.add(ancestor['A'])
ancestor=hierarchy[ancestor['C']]
if ancestor.get('D')=='1':
sector_by_code[node['B']]=ancestor['F']
counts = {}
if a.only_tourism:
db.execute('delete from records where source_hash=?',(SOURCES['tourism.xlsx'][0],))
for alias in (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx')):
source_hash, _, period = SOURCES[alias]
count = 0
province = ''
previous_sheet=''
for sheet, row, c in rows(Path(a.sources)/alias):
if alias == 'tourism.xlsx':
if sheet!=previous_sheet:
province=''
previous_sheet=sheet
heading=norm(c.get('C','')).replace(' ','')
provinces={'cidadedemaputo':'MAPUTO CIDADE','maputocidade':'MAPUTO CIDADE','provinciademaputo':'MAPUTO PROVINCIA','maputoprovincia':'MAPUTO PROVINCIA'}
provinces.update({'provinciade'+norm(n).replace(' ',''):n for n in ['Gaza','Inhambane','Sofala','Manica','Tete','Zambézia','Nampula','Cabo Delgado','Niassa']})
if heading in provinces and not c.get('D'):
province=provinces[heading]
continue
# Explicit province section and ordinal identify an establishment.
# Missing licence/room/contact never excludes an authorised record.
name = c.get('C','')
if not name or not c.get('B','').isdigit():
continue
district = c.get('E','') if sheet=='Folha1' else ''
cae, activity, date, identifier = '', 'Alojamento (âmbito do exemplar; categoria por conferir)', period, ''
sector = labels.get('I','Alojamento e restauração — exemplar de alojamento')
else:
if row == 1:
continue
old = alias == 'companies.xlsx'
name = c.get('L','')
if not name:
continue
province, district = c.get('Y' if old else 'V',''), c.get('X' if old else 'U','')
cae = c.get('S' if old else 'O','')
activity = labels.get(cae, '')
sector = sector_by_code.get(cae,'')
date = c.get('D','')
# Keep source serial verbatim; Excel origin must be demonstrated before conversion.
date = ('Serial Excel original: '+date) if date.replace('.','',1).isdigit() else (date or 'Não indicada')
nuit = c.get('K','')
identifier = ('nuit:'+nuit) if len(nuit)==9 and nuit.isdigit() and len(set(nuit))>1 else ''
rid = digest(source_hash+'|'+sheet+'|'+str(row))
entity = digest(identifier) if identifier else digest('unresolved:'+source_hash+'|'+sheet+'|'+str(row))
# Same projected administrative observation, not byte-identical rows:
# include licence and process identifiers; do not collapse distinct licences.
licence = (c.get('G','') if sheet!='Folha1' else '') if alias=='tourism.xlsx' else c.get('A','')
process = '' if alias=='tourism.xlsx' else c.get('B','')
fp = digest('|'.join(map(norm,[identifier,name,province,district,cae,date,licence,process])))
duplicate = db.execute('select id from records where fingerprint=? and id<>? limit 1',(fp,rid)).fetchone()
db.execute('insert or ignore into records values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', (rid,source_hash,sheet,row,entity,name,province,district,cae,activity,sector,date,fp,duplicate[0] if duplicate else None,0))
count += 1
if count % 10000 == 0:
db.commit()
db.commit()
counts[alias] = count
print(json.dumps({'source':alias,'rows_processed':count}), flush=True)
db.execute('update records set identity_conflict=1 where entity_key in (select entity_key from records group by entity_key having count(distinct lower(name))>1)')
db.commit()
counts={alias:db.execute('select count(*) from records where source_hash=?',(SOURCES[alias][0],)).fetchone()[0] for alias in ('companies.xlsx','licensed.xlsx','tourism.xlsx')}
manifest.update(rows_processed=counts, records=db.execute('select count(*) from records').fetchone()[0],
duplicate_sightings=db.execute('select count(*) from records where duplicate_of is not null').fetchone()[0],
conflicting_identity_sightings=db.execute('select count(*) from records where identity_conflict=1').fetchone()[0],
unclassified_activity=db.execute("select count(*) from records where activity=''").fetchone()[0],
unresolved_territory=db.execute("select count(*) from records where district='' or province=''").fetchone()[0])
db.close()
(output/'preview.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2),encoding='utf-8')
print(json.dumps({k:v for k,v in manifest.items() if k not in ('sources',)},ensure_ascii=False))
if __name__ == '__main__':
main()