137 lines
11 KiB
Python
137 lines
11 KiB
Python
"""Controlled delivery of the accepted internal snapshot. Never publishes entities.
|
|
|
|
Preview is the default. Apply is an authenticated deployment operation, guarded by
|
|
the exact destination, accepted source hashes and a fresh verified backup/restore.
|
|
The rehearsal uses a disposable network-isolated PostgreSQL container only.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
BASELINE = '569a1cd750f3d2a0b87f130e992e73e070bbd1ab'
|
|
RAW_SHA256 = 'dec9db62a52a8bac5f1e2675aecbeb4239f2b1f4fa521db01693c32bdfeef01e'
|
|
ROWS = 361464
|
|
FIELDS = ['id','source_hash','sheet','row_number','entity_key','name','province','district','cae','activity','sector','source_date','fingerprint','duplicate_of','identity_conflict','territory_id','territory_resolution']
|
|
SOURCES = {
|
|
'1ba4c4114b16b407e7746712d00d4f681246bbbc1b42351e96a31f7129acad30':148556,
|
|
'1b5087b464e39e88c42def39655c2391aaaf684b4c832d6a77ff274b98889f00':212631,
|
|
'f02e086ed4973ef17234aa6cc54b98fbb7a6952ffb16ec4dc4f7d0c72af6400c':277,
|
|
'83b4f1200988e99834c2ae3915346fbf237d128308554b67b5a080d4956bf506':0,
|
|
}
|
|
|
|
def csv_value(value):
|
|
if value is None:return r'\N'
|
|
if isinstance(value,bool):value='t' if value else 'f'
|
|
value=str(value)
|
|
if any(c in value for c in ('\x00','\r','\n')):raise ValueError('Unexpected multiline/NUL source projection')
|
|
return '"'+value.replace('"','""')+'"'
|
|
|
|
def fail_unless(ok,message):
|
|
if not ok:raise RuntimeError(message)
|
|
|
|
def sql_command(container=None):
|
|
if container:return ['docker','exec','-i',container,'psql','-p','55441','-U','postgres','-d','proactive_supply_delivery_20261028_test','-XqAt','-v','ON_ERROR_STOP=1']
|
|
return ['runuser','-u','postgres','--','psql','-d','teste_investidor','-XqAt','-v','ON_ERROR_STOP=1']
|
|
|
|
def query(command,text):
|
|
return subprocess.check_output(command,input=text.encode(),stderr=subprocess.PIPE).decode().strip()
|
|
|
|
def prepare(package,output):
|
|
manifest=json.loads((package/'import-manifest.json').read_text())
|
|
preview=json.loads((package/'preview.json').read_text())
|
|
scan=json.loads((package/'scan.json').read_text())
|
|
schema=package/'schema.sql'
|
|
fail_unless(hashlib.file_digest(schema.open('rb'),'sha256').hexdigest()==manifest['schema_sql_sha256'],'Schema SQL differs from reviewed candidate migration')
|
|
fail_unless(manifest['candidate']=='a022ed9848b41ba342b4209414c2bbf2ebd0a7a3','Unexpected candidate')
|
|
fail_unless(set(s['sha256']for s in preview['sources'])==set(SOURCES),'Unexpected originals')
|
|
fail_unless(scan['exit_code']==0 and set(scan['source_hashes'])==set(SOURCES),'Hash-specific source scan receipt missing')
|
|
fail_unless(preview['records']==ROWS,'Incomplete preview')
|
|
fail_unless(hashlib.file_digest((package/'observations.jsonl.gz').open('rb'),'sha256').hexdigest()==manifest['compressed_sha256'],'Compressed snapshot changed')
|
|
rawhash=hashlib.sha256();counts=dict.fromkeys(SOURCES,0);n=0
|
|
seen=set()
|
|
with output.open('w',encoding='utf-8',newline='\n')as sql:
|
|
os.chmod(output,0o600)
|
|
sql.write("BEGIN;\nSET LOCAL lock_timeout='10s';\nSELECT pg_advisory_xact_lock(hashtext('proactive-supply-delivery-v1'));\n")
|
|
sql.write("CREATE TEMP TABLE supply_delivery_sources (LIKE public.proactive_supply_sources INCLUDING DEFAULTS) ON COMMIT DROP;\n")
|
|
sql.write("CREATE TEMP TABLE supply_delivery_observations (LIKE public.proactive_supply_observations INCLUDING DEFAULTS) ON COMMIT DROP;\n")
|
|
sql.write("COPY supply_delivery_sources (sha256,name,period,limitations,receipt,imported_at) FROM STDIN WITH (FORMAT csv,NULL '\\N');\n")
|
|
now=dt.datetime.now(dt.timezone.utc).isoformat()
|
|
for s in preview['sources']:
|
|
receipt={'original':s,'transport_sha256':RAW_SHA256,'candidate':manifest['candidate'],'scan':scan,'access':'internal','publication':False,'counting':'administrative observations;not unique companies;repeated projections retained but excluded from query total'}
|
|
sql.write(','.join(map(csv_value,[s['sha256'],s['name'],s['period'],preview['limitations'],json.dumps(receipt,ensure_ascii=False),now]))+'\n')
|
|
sql.write('\\.\n')
|
|
sql.write("COPY supply_delivery_observations ("+','.join(FIELDS)+") FROM STDIN WITH (FORMAT csv,NULL '\\N');\n")
|
|
with gzip.open(package/'observations.jsonl.gz','rb')as stream:
|
|
for line in stream:
|
|
rawhash.update(line);r=json.loads(line)
|
|
fail_unless(set(r)==set(FIELDS),'Unexpected observation fields')
|
|
fail_unless(r['source_hash']in SOURCES,'Unknown source')
|
|
expected=hashlib.sha256((r['source_hash']+'|'+r['sheet']+'|'+str(r['row_number'])).encode()).hexdigest()
|
|
fail_unless(r['id']==expected and r['id']not in seen,'Invalid or repeated row identity')
|
|
seen.add(r['id']);n+=1;counts[r['source_hash']]+=1
|
|
sql.write(','.join(csv_value(r[k])for k in FIELDS)+'\n')
|
|
sql.write('\\.\n')
|
|
fail_unless(rawhash.hexdigest()==RAW_SHA256 and n==ROWS and counts==SOURCES,'Snapshot checksum/count mismatch')
|
|
sql.write(r'''
|
|
CREATE UNIQUE INDEX supply_delivery_observations_id ON supply_delivery_observations(id);
|
|
ANALYZE supply_delivery_observations;
|
|
DO $$ BEGIN
|
|
IF (SELECT count(*) FROM supply_delivery_observations)<>361464 THEN RAISE EXCEPTION 'Incomplete internal base'; END IF;
|
|
IF (SELECT count(*) FROM supply_delivery_observations WHERE duplicate_of IS NOT NULL)<>147165 THEN RAISE EXCEPTION 'Repeated projection count differs'; END IF;
|
|
IF EXISTS(SELECT 1 FROM supply_delivery_observations x LEFT JOIN supply_delivery_observations y ON y.id=x.duplicate_of WHERE x.duplicate_of IS NOT NULL AND (y.id IS NULL OR x.id=y.id OR x.fingerprint<>y.fingerprint)) THEN RAISE EXCEPTION 'Invalid duplicate lineage'; END IF;
|
|
IF EXISTS(SELECT 1 FROM supply_delivery_observations x LEFT JOIN public.geo_locations g ON g.id=x.territory_id WHERE x.territory_id IS NOT NULL AND (g.id IS NULL OR g.deleted_at IS NOT NULL)) THEN RAISE EXCEPTION 'Destination geography differs'; END IF;
|
|
IF EXISTS(SELECT 1 FROM supply_delivery_observations x JOIN public.proactive_supply_observations y ON x.id=y.id WHERE to_jsonb(x)<>to_jsonb(y)) THEN RAISE EXCEPTION 'Previously installed observation differs;no overwrite'; END IF;
|
|
IF EXISTS(SELECT 1 FROM supply_delivery_sources x JOIN public.proactive_supply_sources y ON x.sha256=y.sha256 WHERE (x.name,x.period,x.limitations) IS DISTINCT FROM (y.name,y.period,y.limitations)) THEN RAISE EXCEPTION 'Existing original metadata differs'; END IF;
|
|
END $$;
|
|
INSERT INTO public.proactive_supply_sources SELECT * FROM supply_delivery_sources ON CONFLICT(sha256) DO NOTHING;
|
|
INSERT INTO public.proactive_supply_observations SELECT * FROM supply_delivery_observations ON CONFLICT(id) DO NOTHING;
|
|
SELECT json_build_object('records',count(*),'distinct_observations',count(*) FILTER(WHERE duplicate_of IS NULL),'repeated_projections',count(*) FILTER(WHERE duplicate_of IS NOT NULL),'not_unique_companies',true,'published',false) FROM public.proactive_supply_observations;
|
|
COMMIT;
|
|
''')
|
|
return {'candidate':manifest['candidate'],'records':n,'source_counts':counts,'raw_sha256':rawhash.hexdigest(),'sql_bytes':output.stat().st_size,'mode':'prepared;no database write','unique_company_count':None,'duplicate_suppression':'WHERE duplicate_of IS NULL;retained administrative sightings do not count as separate offers','national_coverage':'partial','investment_conclusion':'hypothesis only;does not prove demand or viability'}
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--package',type=Path,required=True);p.add_argument('--work',type=Path,required=True);p.add_argument('--mode',choices=['preview','rehearsal','apply'],default='preview');p.add_argument('--container');p.add_argument('--backup',type=Path)
|
|
a=p.parse_args();a.work.mkdir(mode=0o700,parents=True,exist_ok=True);output=a.work/'controlled-import.sql'
|
|
receipt=prepare(a.package,a.work/'controlled-import.sql')
|
|
if a.mode=='preview':print(json.dumps(receipt));return
|
|
fail_unless(os.geteuid()==0,'Authenticated technical deployment identity required')
|
|
if a.mode=='rehearsal':
|
|
fail_unless(a.container is not None,'Disposable QA container required')
|
|
info=json.loads(subprocess.check_output(['docker','inspect',a.container]))[0]
|
|
fail_unless(info['HostConfig']['NetworkMode']=='none' and not info['HostConfig'].get('PortBindings'),'QA must have no network or host ports')
|
|
else:
|
|
fail_unless(not a.container and os.environ.get('SSH_CONNECTION'),'Existing authenticated SSH deployment session required')
|
|
installed=json.loads(Path('/var/lib/investmoz-deploy/installed.json').read_text())
|
|
fail_unless(installed['commit']==BASELINE,'Reconcile installed code before preloading the new private base')
|
|
fail_unless(a.backup is not None,'Fresh backup/restore receipt required')
|
|
base=a.backup.resolve();fail_unless(base.is_relative_to('/var/backups/investmoz-proactive-supply-delivery-20261028'),'Unexpected backup root')
|
|
b=json.loads((base/'backup-receipt.json').read_text());r=json.loads((base/'restore-receipt.json').read_text())
|
|
stamp=dt.datetime.fromisoformat(b['created_at'].replace('Z','+00:00'))
|
|
fail_unless(dt.timedelta(0)<=dt.datetime.now(dt.timezone.utc)-stamp<=dt.timedelta(hours=24),'Backup is no longer fresh;take another before apply')
|
|
fail_unless(b['target']['database']=='teste_investidor' and int(b['target']['port'])==5432,'Backup identity mismatch')
|
|
fail_unless(r['restore_verified'] and r['source_dump_sha256']==b['sha256'],'Restore does not match backup')
|
|
fail_unless(hashlib.file_digest((base/'database.dump').open('rb'),'sha256').hexdigest()==b['sha256'],'Backup bytes changed')
|
|
command=sql_command(a.container)
|
|
identity=query(command,"SELECT current_database()||'|'||current_setting('port');")
|
|
expected='proactive_supply_delivery_20261028_test|55441'if a.mode=='rehearsal'else'teste_investidor|5432'
|
|
fail_unless(identity==expected,'Wrong actual database identity')
|
|
# Existing pipeline does not execute migrations. Schema installation is a
|
|
# separate, reviewed deployment step; this loader never creates it on active.
|
|
fail_unless(query(command,"SELECT count(*) FROM pg_class WHERE oid IN(to_regclass('public.proactive_supply_sources'),to_regclass('public.proactive_supply_observations')) AND relrowsecurity AND relforcerowsecurity;")=='2','Reviewed private schema/RLS not installed')
|
|
before=query(command,'SELECT count(*) FROM public.proactive_supply_observations;')
|
|
with output.open('rb')as stream:
|
|
result=subprocess.run(command,stdin=stream,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
fail_unless(result.returncode==0,'Import rolled back;'+result.stderr.decode(errors='replace')[-1000:])
|
|
lines=[line for line in result.stdout.decode().splitlines()if line.startswith('{')]
|
|
receipt.update(mode=a.mode,database=identity,before=int(before),after=json.loads(lines[-1]),idempotent_no_overwrite=True,active_existing_business_data_touched=False)
|
|
(a.work/'import-receipt.json').write_text(json.dumps(receipt,indent=2));os.chmod(a.work/'import-receipt.json',0o600)
|
|
print(json.dumps(receipt))
|
|
|
|
if __name__=='__main__':main()
|