fix(proactive): retain unnamed administrative rows and preserve accepted import snapshot
This commit is contained in:
@@ -65,6 +65,7 @@ def main():
|
||||
p.add_argument('--sources', required=True)
|
||||
p.add_argument('--output', required=True)
|
||||
p.add_argument('--only-tourism', action='store_true', help='Rebuild only tourism in an existing private preview')
|
||||
p.add_argument('--only-missing-names', action='store_true', help='Add nonempty company rows without a social denomination; never substitute the personal applicant name')
|
||||
a = p.parse_args()
|
||||
output = Path(a.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
@@ -101,7 +102,8 @@ def main():
|
||||
counts = {}
|
||||
if a.only_tourism:
|
||||
db.execute('delete from records where source_hash=?',(SOURCES['tourism.xlsx'][0],))
|
||||
for alias in (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx')):
|
||||
aliases=('companies.xlsx','licensed.xlsx') if a.only_missing_names else (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx'))
|
||||
for alias in aliases:
|
||||
source_hash, _, period = SOURCES[alias]
|
||||
count = 0
|
||||
province = ''
|
||||
@@ -130,8 +132,13 @@ def main():
|
||||
continue
|
||||
old = alias == 'companies.xlsx'
|
||||
name = c.get('L','')
|
||||
if not name:
|
||||
if a.only_missing_names and name:
|
||||
continue
|
||||
if not any(c.values()):
|
||||
continue
|
||||
# A missing social denomination is a data gap, not a reason to
|
||||
# discard an authorised source row or publish an applicant's name.
|
||||
name = name or 'Denominação não indicada'
|
||||
province, district = c.get('Y' if old else 'V',''), c.get('X' if old else 'U','')
|
||||
cae = c.get('S' if old else 'O','')
|
||||
activity = labels.get(cae, '')
|
||||
@@ -156,12 +163,14 @@ def main():
|
||||
db.commit()
|
||||
counts[alias] = count
|
||||
print(json.dumps({'source':alias,'rows_processed':count}), flush=True)
|
||||
db.execute('update records set identity_conflict=1 where entity_key in (select entity_key from records group by entity_key having count(distinct lower(name))>1)')
|
||||
db.execute('update records set identity_conflict=0')
|
||||
db.execute("update records set identity_conflict=1 where entity_key in (select entity_key from records where name<>'Denominação não indicada' group by entity_key having count(distinct lower(name))>1)")
|
||||
db.commit()
|
||||
counts={alias:db.execute('select count(*) from records where source_hash=?',(SOURCES[alias][0],)).fetchone()[0] for alias in ('companies.xlsx','licensed.xlsx','tourism.xlsx')}
|
||||
manifest.update(rows_processed=counts, records=db.execute('select count(*) from records').fetchone()[0],
|
||||
duplicate_sightings=db.execute('select count(*) from records where duplicate_of is not null').fetchone()[0],
|
||||
conflicting_identity_sightings=db.execute('select count(*) from records where identity_conflict=1').fetchone()[0],
|
||||
unnamed_observations=db.execute("select count(*) from records where name='Denominação não indicada'").fetchone()[0],
|
||||
unclassified_activity=db.execute("select count(*) from records where activity=''").fetchone()[0],
|
||||
unresolved_territory=db.execute("select count(*) from records where district='' or province=''").fetchone()[0])
|
||||
db.close()
|
||||
|
||||
Reference in New Issue
Block a user