fix(proactive): retain unnamed administrative rows and preserve accepted import snapshot

This commit is contained in:
2026-10-28 17:28:31 +02:00
parent 2f58b7d438
commit a022ed9848
6 changed files with 111 additions and 36 deletions
+12 -3
View File
@@ -65,6 +65,7 @@ def main():
p.add_argument('--sources', required=True)
p.add_argument('--output', required=True)
p.add_argument('--only-tourism', action='store_true', help='Rebuild only tourism in an existing private preview')
p.add_argument('--only-missing-names', action='store_true', help='Add nonempty company rows without a social denomination; never substitute the personal applicant name')
a = p.parse_args()
output = Path(a.output)
output.mkdir(parents=True, exist_ok=True)
@@ -101,7 +102,8 @@ def main():
counts = {}
if a.only_tourism:
db.execute('delete from records where source_hash=?',(SOURCES['tourism.xlsx'][0],))
for alias in (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx')):
aliases=('companies.xlsx','licensed.xlsx') if a.only_missing_names else (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx'))
for alias in aliases:
source_hash, _, period = SOURCES[alias]
count = 0
province = ''
@@ -130,8 +132,13 @@ def main():
continue
old = alias == 'companies.xlsx'
name = c.get('L','')
if not name:
if a.only_missing_names and name:
continue
if not any(c.values()):
continue
# A missing social denomination is a data gap, not a reason to
# discard an authorised source row or publish an applicant's name.
name = name or 'Denominação não indicada'
province, district = c.get('Y' if old else 'V',''), c.get('X' if old else 'U','')
cae = c.get('S' if old else 'O','')
activity = labels.get(cae, '')
@@ -156,12 +163,14 @@ def main():
db.commit()
counts[alias] = count
print(json.dumps({'source':alias,'rows_processed':count}), flush=True)
db.execute('update records set identity_conflict=1 where entity_key in (select entity_key from records group by entity_key having count(distinct lower(name))>1)')
db.execute('update records set identity_conflict=0')
db.execute("update records set identity_conflict=1 where entity_key in (select entity_key from records where name<>'Denominação não indicada' group by entity_key having count(distinct lower(name))>1)")
db.commit()
counts={alias:db.execute('select count(*) from records where source_hash=?',(SOURCES[alias][0],)).fetchone()[0] for alias in ('companies.xlsx','licensed.xlsx','tourism.xlsx')}
manifest.update(rows_processed=counts, records=db.execute('select count(*) from records').fetchone()[0],
duplicate_sightings=db.execute('select count(*) from records where duplicate_of is not null').fetchone()[0],
conflicting_identity_sightings=db.execute('select count(*) from records where identity_conflict=1').fetchone()[0],
unnamed_observations=db.execute("select count(*) from records where name='Denominação não indicada'").fetchone()[0],
unclassified_activity=db.execute("select count(*) from records where activity=''").fetchone()[0],
unresolved_territory=db.execute("select count(*) from records where district='' or province=''").fetchone()[0])
db.close()