fix(proactive): retain unnamed administrative rows and preserve accepted import snapshot
This commit is contained in:
@@ -65,6 +65,7 @@ def main():
|
||||
p.add_argument('--sources', required=True)
|
||||
p.add_argument('--output', required=True)
|
||||
p.add_argument('--only-tourism', action='store_true', help='Rebuild only tourism in an existing private preview')
|
||||
p.add_argument('--only-missing-names', action='store_true', help='Add nonempty company rows without a social denomination; never substitute the personal applicant name')
|
||||
a = p.parse_args()
|
||||
output = Path(a.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
@@ -101,7 +102,8 @@ def main():
|
||||
counts = {}
|
||||
if a.only_tourism:
|
||||
db.execute('delete from records where source_hash=?',(SOURCES['tourism.xlsx'][0],))
|
||||
for alias in (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx')):
|
||||
aliases=('companies.xlsx','licensed.xlsx') if a.only_missing_names else (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx'))
|
||||
for alias in aliases:
|
||||
source_hash, _, period = SOURCES[alias]
|
||||
count = 0
|
||||
province = ''
|
||||
@@ -130,8 +132,13 @@ def main():
|
||||
continue
|
||||
old = alias == 'companies.xlsx'
|
||||
name = c.get('L','')
|
||||
if not name:
|
||||
if a.only_missing_names and name:
|
||||
continue
|
||||
if not any(c.values()):
|
||||
continue
|
||||
# A missing social denomination is a data gap, not a reason to
|
||||
# discard an authorised source row or publish an applicant's name.
|
||||
name = name or 'Denominação não indicada'
|
||||
province, district = c.get('Y' if old else 'V',''), c.get('X' if old else 'U','')
|
||||
cae = c.get('S' if old else 'O','')
|
||||
activity = labels.get(cae, '')
|
||||
@@ -156,12 +163,14 @@ def main():
|
||||
db.commit()
|
||||
counts[alias] = count
|
||||
print(json.dumps({'source':alias,'rows_processed':count}), flush=True)
|
||||
db.execute('update records set identity_conflict=1 where entity_key in (select entity_key from records group by entity_key having count(distinct lower(name))>1)')
|
||||
db.execute('update records set identity_conflict=0')
|
||||
db.execute("update records set identity_conflict=1 where entity_key in (select entity_key from records where name<>'Denominação não indicada' group by entity_key having count(distinct lower(name))>1)")
|
||||
db.commit()
|
||||
counts={alias:db.execute('select count(*) from records where source_hash=?',(SOURCES[alias][0],)).fetchone()[0] for alias in ('companies.xlsx','licensed.xlsx','tourism.xlsx')}
|
||||
manifest.update(rows_processed=counts, records=db.execute('select count(*) from records').fetchone()[0],
|
||||
duplicate_sightings=db.execute('select count(*) from records where duplicate_of is not null').fetchone()[0],
|
||||
conflicting_identity_sightings=db.execute('select count(*) from records where identity_conflict=1').fetchone()[0],
|
||||
unnamed_observations=db.execute("select count(*) from records where name='Denominação não indicada'").fetchone()[0],
|
||||
unclassified_activity=db.execute("select count(*) from records where activity=''").fetchone()[0],
|
||||
unresolved_territory=db.execute("select count(*) from records where district='' or province=''").fetchone()[0])
|
||||
db.close()
|
||||
|
||||
@@ -23,7 +23,7 @@ if($operation==='schema'){
|
||||
}
|
||||
echo "QA schema available; no existing table altered.\n";exit;
|
||||
}
|
||||
if($operation==='import'){
|
||||
if(in_array($operation,['import','extend'],true)){
|
||||
$dir=$out.'/private-v2';
|
||||
$manifest=json_decode(file_get_contents($dir.'/preview.json'),true,512,JSON_THROW_ON_ERROR);
|
||||
$scan=json_decode(file_get_contents($out.'/private/scan.json'),true,512,JSON_THROW_ON_ERROR);
|
||||
@@ -31,11 +31,25 @@ if($operation==='import'){
|
||||
$file=$dir.'/observations.jsonl';$hash=hash_file('sha256',$file);
|
||||
if(($argv[2]??'')!==$hash)throw new RuntimeException('Review preview and supply exact JSONL hash');
|
||||
$before=DB::table('proactive_supply_observations')->count();
|
||||
DB::transaction(function()use($manifest,$file,$hash){
|
||||
$extension=$operation==='extend';$previousHash=$argv[3]??null;
|
||||
if($extension){
|
||||
$delta=json_decode(file_get_contents($dir.'/extension-preview.json'),true,512,JSON_THROW_ON_ERROR);
|
||||
if(($delta['sha256']??null)!==$hash||($delta['previous_sha256']??null)!==$previousHash||empty($delta['no_replacement'])||$delta['previous_observations_unchanged']+$delta['new_observations']!==$manifest['records'])throw new RuntimeException('Extension must prove all previously accepted observations unchanged and bind both snapshot hashes.');
|
||||
}
|
||||
DB::transaction(function()use($manifest,$file,$hash,$extension,$previousHash){
|
||||
DB::select("select pg_advisory_xact_lock(hashtext('proactive-supply-import-v1'))");
|
||||
foreach($manifest['sources']as$s){
|
||||
$existing=DB::table('proactive_supply_sources')->where('sha256',$s['sha256'])->first();
|
||||
if($existing && data_get(json_decode($existing->receipt,true),'transport_sha256')!==$hash)throw new RuntimeException('Existing source has a different prepared classification snapshot; preserve it and prepare an explicit mapping delta instead of silently replacing observations.');
|
||||
if($existing && data_get(json_decode($existing->receipt,true),'transport_sha256')!==($extension?$previousHash:$hash))throw new RuntimeException('Existing source has a different prepared classification snapshot; preserve it and prepare an explicit mapping delta instead of silently replacing observations.');
|
||||
if($extension){
|
||||
if(!$existing)throw new RuntimeException('An extension must reference the already imported originals.');
|
||||
$receipt=json_decode($existing->receipt,true);
|
||||
$history=$receipt['extensions']??[];
|
||||
if(!in_array($hash,array_column($history,'transport_sha256'),true)){
|
||||
$history[]=['transport_sha256'=>$hash,'rows_in_extended_preview'=>$manifest['records'],'at'=>now()->toIso8601String(),'operation'=>'append previously unrepresented source rows;existing observations immutable'];
|
||||
DB::table('proactive_supply_sources')->where('sha256',$s['sha256'])->update(['receipt'=>json_encode([...$receipt,'extensions'=>$history])]);
|
||||
}
|
||||
}
|
||||
DB::table('proactive_supply_sources')->insertOrIgnore(['sha256'=>$s['sha256'],'name'=>$s['name'],'period'=>$s['period'],'limitations'=>$manifest['limitations'],'receipt'=>json_encode(['source'=>$s,'preview_rows'=>$manifest['records'],'transport_sha256'=>$hash,'scan'=>'Defender custom scan, no threats, hash-specific receipt','scope'=>'isolated QA, internal analysis only']),'imported_at'=>now()]);
|
||||
}
|
||||
$f=fopen($file,'rb');$batch=[];$n=0;
|
||||
@@ -48,7 +62,7 @@ if($operation==='import'){
|
||||
fclose($f);if($batch)DB::table('proactive_supply_observations')->insertOrIgnore($batch);
|
||||
if($n!==$manifest['records'])throw new RuntimeException('Incomplete candidate');
|
||||
});
|
||||
$receipt=['before'=>$before,'after'=>DB::table('proactive_supply_observations')->count(),'sha256'=>$hash,'publication'=>false,'database'=>'proactive_investment_20261027_test','port'=>55441];
|
||||
file_put_contents($out.'/import-'.($before?'replay':'first').'.json',json_encode($receipt,JSON_PRETTY_PRINT));echo json_encode($receipt)."\n";exit;
|
||||
$receipt=['before'=>$before,'after'=>DB::table('proactive_supply_observations')->count(),'sha256'=>$hash,'extension'=>$extension,'previous_snapshot'=>$previousHash,'publication'=>false,'database'=>'proactive_investment_20261027_test','port'=>55441];
|
||||
file_put_contents($out.'/import-'.($extension?'extension':($before?'replay':'first')).'.json',json_encode($receipt,JSON_PRETTY_PRINT));echo json_encode($receipt)."\n";exit;
|
||||
}
|
||||
throw new RuntimeException('Use geography, schema, import <reviewed hash>, or test; exclusively isolated QA.');
|
||||
throw new RuntimeException('Use geography,schema,import <reviewed hash>,extend <reviewed hash> <previous snapshot hash>,or test;exclusively isolated QA.');
|
||||
|
||||
Reference in New Issue
Block a user