feat(proactive): integrate private supply observations and demonstrate Gurue comparison in QA
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""Map only unique canonical names within the same province, then export privately."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import unicodedata
|
||||
|
||||
def norm(v):
|
||||
return ' '.join(''.join(c for c in unicodedata.normalize('NFKD',v).casefold() if not unicodedata.combining(c)).split())
|
||||
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument('--preview',required=True)
|
||||
p.add_argument('--geography',required=True)
|
||||
a=p.parse_args()
|
||||
root=Path(a.preview)
|
||||
geos=json.loads(Path(a.geography).read_text(encoding='utf-8'))
|
||||
byid={r['id']:r for r in geos}
|
||||
territories={}
|
||||
provinces={}
|
||||
subterritories={}
|
||||
def province_name(v):
|
||||
n=norm(v)
|
||||
return 'maputo provincia' if n=='maputo' else n
|
||||
for g in geos:
|
||||
if g['location_type']=='province':provinces.setdefault(norm(g['name']),[]).append(g['id'])
|
||||
if g['location_type'] not in ('district','admin_post','locality'):continue
|
||||
parent=byid.get(g['parent_id']);visited=set()
|
||||
while parent and parent['location_type']!='province' and parent['id'] not in visited:
|
||||
visited.add(parent['id']);parent=byid.get(parent['parent_id'])
|
||||
if not parent:continue
|
||||
target=territories if g['location_type']=='district' else subterritories
|
||||
target.setdefault((norm(parent['name']),norm(g['name'])),[]).append(g['id'])
|
||||
db=sqlite3.connect(root/'supply-preview.sqlite')
|
||||
db.row_factory=sqlite3.Row
|
||||
resolution={}
|
||||
with (root/'observations.jsonl').open('w',encoding='utf-8',newline='\n') as f:
|
||||
for row in db.execute('select * from records order by source_hash,sheet,row_number'):
|
||||
r=dict(row)
|
||||
key=(province_name(r['province']),norm(r['district']))
|
||||
hits=territories.get(key,[])
|
||||
lower_level=False
|
||||
if not hits:
|
||||
hits=subterritories.get(key,[])
|
||||
lower_level=bool(hits)
|
||||
province_hits=provinces.get(province_name(r['province']),[])
|
||||
province_only=not r['district'] and len(province_hits)==1
|
||||
r['territory_id']=hits[0] if len(hits)==1 else (province_hits[0] if province_only else None)
|
||||
r['territory_resolution']=('exact_normalized_province_subterritory' if lower_level else 'exact_normalized_province_district') if len(hits)==1 else ('province_only_district_unknown' if province_only else ('ambiguous' if hits else 'unresolved'))
|
||||
r['identity_conflict']=bool(r['identity_conflict'])
|
||||
resolution[r['territory_resolution']]=resolution.get(r['territory_resolution'],0)+1
|
||||
f.write(json.dumps(r,ensure_ascii=False)+'\n')
|
||||
result={'rows':db.execute('select count(*) from records').fetchone()[0],'territory_resolution':resolution,'sha256':hashlib.file_digest((root/'observations.jsonl').open('rb'),'sha256').hexdigest(),'mapping':'case/accent/whitespace; exact district in province, otherwise unique named subordinate territory within that province; source label Maputo distinct from Maputo Cidade maps explicitly to Maputo Província; city names retain canonical administrative level, no fuzzy matching','publication':False}
|
||||
(root/'export.json').write_text(json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
print(json.dumps(result,ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Read-only XLSX ingestion into a private, indexed analysis snapshot. No publication.
|
||||
|
||||
Every source row remains addressable. Matching identifiers suggest the same entity;
|
||||
they never imply a current trading status, contract, or an exhaustive census.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import unicodedata
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
|
||||
NS = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
|
||||
SOURCES = {
|
||||
'companies.xlsx': ('1ba4c4114b16b407e7746712d00d4f681246bbbc1b42351e96a31f7129acad30', 'EMPRESAS_LIST 2014-2022.xlsx', '2014–2022; data por registo quando disponível'),
|
||||
'licensed.xlsx': ('1b5087b464e39e88c42def39655c2391aaaf684b4c832d6a77ff274b98889f00', 'Empresas Licenciadas.xlsx', 'Data por registo; cobertura temporal global não determinada'),
|
||||
'tourism.xlsx': ('f02e086ed4973ef17234aa6cc54b98fbb7a6952ffb16ec4dc4f7d0c72af6400c', 'CADASTRO DE EMPREENDIMENTOS TURISTICOS 3, 4 E 5 ESTRELAS 30.10.24 18,00h.xlsx', '30/10/2024 (nome do exemplar); situação actual não confirmada'),
|
||||
'cae.xlsx': ('83b4f1200988e99834c2ae3915346fbf237d128308554b67b5a080d4956bf506', 'CAE eBAU.xlsx', 'Versão não indicada'),
|
||||
}
|
||||
|
||||
def norm(v):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD', str(v)).casefold() if not unicodedata.combining(c)).strip()
|
||||
|
||||
def digest(v):
|
||||
return hashlib.sha256(v.encode('utf-8')).hexdigest()
|
||||
|
||||
def rows(path):
|
||||
# Shared strings are spooled to a temporary on-disk SQLite lookup, not a huge XML tree.
|
||||
with zipfile.ZipFile(path) as z, sqlite3.connect('') as strings:
|
||||
strings.execute('create table strings(id integer primary key,value text)')
|
||||
if 'xl/sharedStrings.xml' in z.namelist():
|
||||
with z.open('xl/sharedStrings.xml') as f:
|
||||
i = 0
|
||||
for _, e in ET.iterparse(f, events=('end',)):
|
||||
if e.tag == NS+'si':
|
||||
strings.execute('insert into strings values(?,?)', (i, ''.join(t.text or '' for t in e.iter(NS+'t'))))
|
||||
i += 1
|
||||
e.clear()
|
||||
book = ET.fromstring(z.read('xl/workbook.xml'))
|
||||
rels = {r.attrib['Id']: r.attrib['Target'] for r in ET.fromstring(z.read('xl/_rels/workbook.xml.rels'))}
|
||||
for sheet in book.find(NS+'sheets'):
|
||||
target = rels[sheet.attrib['{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id']]
|
||||
target = target.lstrip('/') if target.startswith('/') else 'xl/'+target
|
||||
with z.open(target) as f:
|
||||
for _, e in ET.iterparse(f, events=('end',)):
|
||||
if e.tag != NS+'row':
|
||||
continue
|
||||
result = {}
|
||||
for c in e:
|
||||
col = ''.join(ch for ch in c.attrib.get('r','') if ch.isalpha())
|
||||
v = c.findtext(NS+'v', '')
|
||||
if c.attrib.get('t') == 's':
|
||||
hit = strings.execute('select value from strings where id=?', (int(v),)).fetchone()
|
||||
v = hit[0] if hit else ''
|
||||
elif c.attrib.get('t') == 'inlineStr':
|
||||
v = ''.join(t.text or '' for t in c.iter(NS+'t'))
|
||||
result[col] = ' '.join(v.split())
|
||||
yield sheet.attrib['name'], int(e.attrib['r']), result
|
||||
e.clear()
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--sources', required=True)
|
||||
p.add_argument('--output', required=True)
|
||||
p.add_argument('--only-tourism', action='store_true', help='Rebuild only tourism in an existing private preview')
|
||||
a = p.parse_args()
|
||||
output = Path(a.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {'version':1, 'access':'internal', 'sources':[], 'publication':False,
|
||||
'limitations':'Registos administrativos históricos; cobertura incompleta, actividade actual, capacidade, procura e compromissos não confirmados. Ausência de registo significa informação insuficiente.'}
|
||||
for alias, (expected, name, period) in SOURCES.items():
|
||||
path = Path(a.sources)/alias
|
||||
actual = hashlib.file_digest(path.open('rb'), 'sha256').hexdigest()
|
||||
if actual != expected:
|
||||
raise ValueError('Original changed: '+alias)
|
||||
manifest['sources'].append({'alias':alias,'name':name,'sha256':actual,'bytes':path.stat().st_size,'period':period})
|
||||
db = sqlite3.connect(output/'supply-preview.sqlite')
|
||||
db.executescript('''create table if not exists records(id text primary key, source_hash text, sheet text, row_number integer, entity_key text, name text, province text, district text, cae text, activity text, sector text, source_date text, fingerprint text, duplicate_of text, identity_conflict integer default 0);
|
||||
create index if not exists supply_place on records(province,district);
|
||||
create index if not exists supply_entity on records(entity_key);
|
||||
create index if not exists supply_fingerprint on records(fingerprint);''')
|
||||
labels = {}
|
||||
hierarchy = {}
|
||||
for _, row, c in rows(Path(a.sources)/'cae.xlsx'):
|
||||
if row > 1 and c.get('B') and c.get('F'):
|
||||
labels[c['B']] = c['F']
|
||||
hierarchy[c['A']] = c
|
||||
# Supplied CAE hierarchy labels retained. Sector is the top-level CAE section,
|
||||
# not an invented correspondence to the portal's twelve navigation sectors.
|
||||
sector_by_code = {}
|
||||
for node in hierarchy.values():
|
||||
ancestor=node
|
||||
visited=set()
|
||||
while ancestor.get('D')!='1' and ancestor.get('C') in hierarchy and ancestor['A'] not in visited:
|
||||
visited.add(ancestor['A'])
|
||||
ancestor=hierarchy[ancestor['C']]
|
||||
if ancestor.get('D')=='1':
|
||||
sector_by_code[node['B']]=ancestor['F']
|
||||
counts = {}
|
||||
if a.only_tourism:
|
||||
db.execute('delete from records where source_hash=?',(SOURCES['tourism.xlsx'][0],))
|
||||
for alias in (('tourism.xlsx',) if a.only_tourism else ('companies.xlsx','licensed.xlsx','tourism.xlsx')):
|
||||
source_hash, _, period = SOURCES[alias]
|
||||
count = 0
|
||||
province = ''
|
||||
previous_sheet=''
|
||||
for sheet, row, c in rows(Path(a.sources)/alias):
|
||||
if alias == 'tourism.xlsx':
|
||||
if sheet!=previous_sheet:
|
||||
province=''
|
||||
previous_sheet=sheet
|
||||
heading=norm(c.get('C','')).replace(' ','')
|
||||
provinces={'cidadedemaputo':'MAPUTO CIDADE','maputocidade':'MAPUTO CIDADE','provinciademaputo':'MAPUTO PROVINCIA','maputoprovincia':'MAPUTO PROVINCIA'}
|
||||
provinces.update({'provinciade'+norm(n).replace(' ',''):n for n in ['Gaza','Inhambane','Sofala','Manica','Tete','Zambézia','Nampula','Cabo Delgado','Niassa']})
|
||||
if heading in provinces and not c.get('D'):
|
||||
province=provinces[heading]
|
||||
continue
|
||||
# Explicit province section and ordinal identify an establishment.
|
||||
# Missing licence/room/contact never excludes an authorised record.
|
||||
name = c.get('C','')
|
||||
if not name or not c.get('B','').isdigit():
|
||||
continue
|
||||
district = c.get('E','') if sheet=='Folha1' else ''
|
||||
cae, activity, date, identifier = '', 'Alojamento (âmbito do exemplar; categoria por conferir)', period, ''
|
||||
sector = labels.get('I','Alojamento e restauração — exemplar de alojamento')
|
||||
else:
|
||||
if row == 1:
|
||||
continue
|
||||
old = alias == 'companies.xlsx'
|
||||
name = c.get('L','')
|
||||
if not name:
|
||||
continue
|
||||
province, district = c.get('Y' if old else 'V',''), c.get('X' if old else 'U','')
|
||||
cae = c.get('S' if old else 'O','')
|
||||
activity = labels.get(cae, '')
|
||||
sector = sector_by_code.get(cae,'')
|
||||
date = c.get('D','')
|
||||
# Keep source serial verbatim; Excel origin must be demonstrated before conversion.
|
||||
date = ('Serial Excel original: '+date) if date.replace('.','',1).isdigit() else (date or 'Não indicada')
|
||||
nuit = c.get('K','')
|
||||
identifier = ('nuit:'+nuit) if len(nuit)==9 and nuit.isdigit() and len(set(nuit))>1 else ''
|
||||
rid = digest(source_hash+'|'+sheet+'|'+str(row))
|
||||
entity = digest(identifier) if identifier else digest('unresolved:'+source_hash+'|'+sheet+'|'+str(row))
|
||||
# Same projected administrative observation, not byte-identical rows:
|
||||
# include licence and process identifiers; do not collapse distinct licences.
|
||||
licence = (c.get('G','') if sheet!='Folha1' else '') if alias=='tourism.xlsx' else c.get('A','')
|
||||
process = '' if alias=='tourism.xlsx' else c.get('B','')
|
||||
fp = digest('|'.join(map(norm,[identifier,name,province,district,cae,date,licence,process])))
|
||||
duplicate = db.execute('select id from records where fingerprint=? and id<>? limit 1',(fp,rid)).fetchone()
|
||||
db.execute('insert or ignore into records values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', (rid,source_hash,sheet,row,entity,name,province,district,cae,activity,sector,date,fp,duplicate[0] if duplicate else None,0))
|
||||
count += 1
|
||||
if count % 10000 == 0:
|
||||
db.commit()
|
||||
db.commit()
|
||||
counts[alias] = count
|
||||
print(json.dumps({'source':alias,'rows_processed':count}), flush=True)
|
||||
db.execute('update records set identity_conflict=1 where entity_key in (select entity_key from records group by entity_key having count(distinct lower(name))>1)')
|
||||
db.commit()
|
||||
counts={alias:db.execute('select count(*) from records where source_hash=?',(SOURCES[alias][0],)).fetchone()[0] for alias in ('companies.xlsx','licensed.xlsx','tourism.xlsx')}
|
||||
manifest.update(rows_processed=counts, records=db.execute('select count(*) from records').fetchone()[0],
|
||||
duplicate_sightings=db.execute('select count(*) from records where duplicate_of is not null').fetchone()[0],
|
||||
conflicting_identity_sightings=db.execute('select count(*) from records where identity_conflict=1').fetchone()[0],
|
||||
unclassified_activity=db.execute("select count(*) from records where activity=''").fetchone()[0],
|
||||
unresolved_territory=db.execute("select count(*) from records where district='' or province=''").fetchone()[0])
|
||||
db.close()
|
||||
(output/'preview.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
print(json.dumps({k:v for k,v in manifest.items() if k not in ('sources',)},ensure_ascii=False))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
$root=dirname(__DIR__);
|
||||
$operation='test';
|
||||
if(!in_array($operation,['routes','test'],true)||basename($root)!=='portal-investidor-supply-qa-20261028')throw new RuntimeException('Explicit isolated QA workspace and routes/test operation required.');
|
||||
foreach(['storage/framework/views','storage/framework/cache','storage/framework/sessions','bootstrap/cache','storage/logs','outputs/proactive-supply-20261028','outputs/proactive-investment-20261027']as$dir)if(!is_dir($root.'/'.$dir))mkdir($root.'/'.$dir,0700,true);
|
||||
foreach(['APP_BASE_PATH'=>$root,'APP_ENV'=>'testing','APP_URL'=>'http://127.0.0.1:18768','APP_KEY'=>'base64:'.base64_encode(str_repeat('Q',32)),'DB_CONNECTION'=>'pgsql','DB_HOST'=>'127.0.0.1','DB_PORT'=>'55441','DB_DATABASE'=>'proactive_investment_20261027_test','DB_USERNAME'=>'postgres','DB_PASSWORD'=>'','DB_URL'=>'','CACHE_STORE'=>'array','SESSION_DRIVER'=>'array','QUEUE_CONNECTION'=>'sync','MAIL_MAILER'=>'array'] as $k=>$v){putenv($k.'='.$v);$_ENV[$k]=$_SERVER[$k]=$v;}
|
||||
$loader=require $root.'/vendor/autoload.php';
|
||||
spl_autoload_register(function($class)use($root){foreach(['App\\'=>'app/','Tests\\'=>'tests/','Database\\Factories\\'=>'database/factories/']as$prefix=>$dir){if(str_starts_with($class,$prefix)){$file=$root.'/'.$dir.str_replace('\\','/',substr($class,strlen($prefix))).'.php';if(is_file($file)){require $file;return;}}}},true,true);
|
||||
$app=require $root.'/bootstrap/app.php';
|
||||
(new Illuminate\Foundation\Bootstrap\LoadEnvironmentVariables)->bootstrap($app);
|
||||
(new Illuminate\Foundation\Bootstrap\LoadConfiguration)->bootstrap($app);
|
||||
$c=config('database.connections.pgsql');
|
||||
if($app->configurationIsCached()||$c['host']!=='127.0.0.1'||(int)$c['port']!==55441||$c['database']!=='proactive_investment_20261027_test'||!empty($c['url']))throw new RuntimeException('QA guard');
|
||||
$pdo=new PDO('pgsql:host=127.0.0.1;port=55441;dbname='.$c['database'],$c['username'],$c['password']);
|
||||
if($pdo->query('select current_database()')->fetchColumn()!==$c['database'])throw new RuntimeException('QA identity');
|
||||
|
||||
if(($argv[1]??'')==='test')return;
|
||||
$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap();
|
||||
config(['portal.profile_access_enabled'=>true,'portal.opportunity_specialized_drafts_enabled'=>true,'investor-path.enabled'=>true,'investor-path.private_preparation'=>true,'suitecrm.enabled'=>false,'investmoz_ai.enabled'=>false,'session.driver'=>'file','session.secure'=>false]);
|
||||
|
||||
Illuminate\Support\Facades\Vite::useBuildDirectory('build-qa-opportunities');
|
||||
config(['session.cookie'=>'proactive_qa_session','session.http_only'=>true]);
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
/** Explicit QA-only tooling: no active environment or .env fallback. */
|
||||
require __DIR__.'/proactive-supply-qa-bootstrap.php';
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
$operation=$argv[1]??'';
|
||||
if($operation==='test'){
|
||||
$argv=['phpunit','-c','phpunit.investmoz.xml','tests/Feature/ProactiveSupplyTest.php','tests/Feature/ProactiveInvestmentTest.php'];
|
||||
if(getenv('PROACTIVE_SUPPLY_FILTER')){$argv[]='--filter';$argv[]=getenv('PROACTIVE_SUPPLY_FILTER');}
|
||||
$argc=count($argv);$_SERVER['argv']=$argv;$_SERVER['argc']=$argc;
|
||||
require dirname(__DIR__).'/vendor/phpunit/phpunit/phpunit';exit;
|
||||
}
|
||||
Tests\Support\ProactiveInvestmentFixture::guard();
|
||||
$out=dirname(__DIR__).'/outputs/proactive-supply-20261028';
|
||||
if($operation==='geography'){
|
||||
file_put_contents($out.'/private/geography.json',json_encode(DB::table('geo_locations')->whereNull('deleted_at')->get(['id','name','parent_id','location_type']),JSON_UNESCAPED_UNICODE|JSON_THROW_ON_ERROR));
|
||||
echo "QA geography exported privately.\n";exit;
|
||||
}
|
||||
if($operation==='schema'){
|
||||
if(!Schema::hasTable('proactive_supply_sources')){
|
||||
(require dirname(__DIR__).'/database/migrations/2026_10_28_180000_create_proactive_supply_observations.php')->up();
|
||||
}
|
||||
echo "QA schema available; no existing table altered.\n";exit;
|
||||
}
|
||||
if($operation==='import'){
|
||||
$dir=$out.'/private-v2';
|
||||
$manifest=json_decode(file_get_contents($dir.'/preview.json'),true,512,JSON_THROW_ON_ERROR);
|
||||
$scan=json_decode(file_get_contents($out.'/private/scan.json'),true,512,JSON_THROW_ON_ERROR);
|
||||
if(($scan['exit_code']??1)!==0||array_diff(array_column($manifest['sources'],'sha256'),$scan['source_hashes']??[]))throw new RuntimeException('Hash-specific malware scan required');
|
||||
$file=$dir.'/observations.jsonl';$hash=hash_file('sha256',$file);
|
||||
if(($argv[2]??'')!==$hash)throw new RuntimeException('Review preview and supply exact JSONL hash');
|
||||
$before=DB::table('proactive_supply_observations')->count();
|
||||
DB::transaction(function()use($manifest,$file,$hash){
|
||||
DB::select("select pg_advisory_xact_lock(hashtext('proactive-supply-import-v1'))");
|
||||
foreach($manifest['sources']as$s){
|
||||
$existing=DB::table('proactive_supply_sources')->where('sha256',$s['sha256'])->first();
|
||||
if($existing && data_get(json_decode($existing->receipt,true),'transport_sha256')!==$hash)throw new RuntimeException('Existing source has a different prepared classification snapshot; preserve it and prepare an explicit mapping delta instead of silently replacing observations.');
|
||||
DB::table('proactive_supply_sources')->insertOrIgnore(['sha256'=>$s['sha256'],'name'=>$s['name'],'period'=>$s['period'],'limitations'=>$manifest['limitations'],'receipt'=>json_encode(['source'=>$s,'preview_rows'=>$manifest['records'],'transport_sha256'=>$hash,'scan'=>'Defender custom scan, no threats, hash-specific receipt','scope'=>'isolated QA, internal analysis only']),'imported_at'=>now()]);
|
||||
}
|
||||
$f=fopen($file,'rb');$batch=[];$n=0;
|
||||
while(($line=fgets($f))!==false){
|
||||
$r=json_decode($line,true,512,JSON_THROW_ON_ERROR);
|
||||
if(!in_array($r['source_hash'],array_column($manifest['sources'],'sha256'),true))throw new RuntimeException('Unknown source');
|
||||
$batch[]=$r;$n++;
|
||||
if(count($batch)===500){DB::table('proactive_supply_observations')->insertOrIgnore($batch);$batch=[];}
|
||||
}
|
||||
fclose($f);if($batch)DB::table('proactive_supply_observations')->insertOrIgnore($batch);
|
||||
if($n!==$manifest['records'])throw new RuntimeException('Incomplete candidate');
|
||||
});
|
||||
$receipt=['before'=>$before,'after'=>DB::table('proactive_supply_observations')->count(),'sha256'=>$hash,'publication'=>false,'database'=>'proactive_investment_20261027_test','port'=>55441];
|
||||
file_put_contents($out.'/import-'.($before?'replay':'first').'.json',json_encode($receipt,JSON_PRETTY_PRINT));echo json_encode($receipt)."\n";exit;
|
||||
}
|
||||
throw new RuntimeException('Use geography, schema, import <reviewed hash>, or test; exclusively isolated QA.');
|
||||
Reference in New Issue
Block a user