fix(demography): verify D01-D06 in isolated QA with INE traceability

This commit is contained in:
2026-11-20 20:07:16 +02:00
parent 558c11ef17
commit 3f66201cbe
17 changed files with 570 additions and 89 deletions
+25
View File
@@ -0,0 +1,25 @@
<?php
// Read-only comparison of QA observations with exact INE slices.
require __DIR__.'/continuity-qa-bootstrap.php';
$app = require dirname(__DIR__).'/bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\DB;
$identity = DB::selectOne('select current_database() as name, inet_server_port() as port');
throw_unless($identity->name === 'continuity_20261120_test' && (int) $identity->port === 55441, RuntimeException::class, 'Isolated QA only');
$slices = json_decode(file_get_contents(dirname(__DIR__).'/outputs/demography-d01-d06-20261120/source-slices.json'), true, 512, JSON_THROW_ON_ERROR);
$checked = 0;
foreach ($slices as $slice) {
$batch = DB::table('demographic_imports')->where('sha256', $slice['sha256'])->first();
throw_unless($batch && $batch->file_name === $slice['file'], RuntimeException::class, 'Original provenance missing');
foreach ($slice['records'] as $row) {
$stored = DB::table('demographic_observations as o')->join('geo_locations as g', 'g.id', '=', 'o.geo_location_id')
->where('o.import_id', $batch->id)->where('o.sheet', $row['sheet'])->where('o.cell', $row['cell'])->first(['o.*', 'g.name']);
throw_unless($stored && $stored->name === $row['territory'] && $stored->numeric_value === $row['numeric_value'] && $stored->original_value === $row['original_value'], RuntimeException::class, 'Observation mismatch');
$checked++;
}
}
echo json_encode(['database' => $identity->name, 'port' => $identity->port, 'exact_observations_checked' => $checked, 'mismatches' => 0, 'writes' => false, 'traceability_preserved' => true], JSON_PRETTY_PRINT).PHP_EOL;
+49
View File
@@ -0,0 +1,49 @@
<?php
// QA fixture: exact read-only INE slices, synthetic territorial identities and no opportunities.
require __DIR__.'/continuity-qa-bootstrap.php';
$app = require dirname(__DIR__).'/bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str;
$identity = DB::selectOne('select current_database() as name, inet_server_port() as port');
throw_unless($identity->name === 'continuity_20261120_test' && (int) $identity->port === 55441, RuntimeException::class, 'Isolated QA only');
$slices = json_decode(file_get_contents(dirname(__DIR__).'/outputs/demography-d01-d06-20261120/source-slices.json'), true, 512, JSON_THROW_ON_ERROR);
$result = DB::transaction(function () use ($slices) {
$actor = DB::table('users')->first();
throw_unless($actor, RuntimeException::class, 'Existing synthetic investor required');
foreach (['comercio-e-servicos' => 'Comércio e serviços', 'turismo' => 'Turismo'] as $slug => $name) {
DB::table('sectors')->insertOrIgnore(['slug' => $slug, 'name' => $name, 'is_active' => true, 'published_at' => now(), 'created_at' => now(), 'updated_at' => now()]);
}
$country = DB::table('geo_locations')->where('location_type', 'country')->value('id');
$result = [];
foreach ($slices as $slice) {
$provinceName = $slice['records'][0]['province'];
$territoryName = $slice['records'][0]['territory'];
foreach ([['province', $provinceName], ['district', $territoryName]] as [$level, $name]) {
$id = (string) Str::uuid();
$external = '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff:D01-D06:'.$name;
DB::table('geo_locations')->insertOrIgnore(['id' => $id, 'parent_id' => $level === 'province' ? $country : $province, 'name' => $name, 'code' => 'QA-D01-'.$name, 'location_type' => $level, 'source_system' => 'cartografia-workbook', 'external_id' => $external, 'data_classification' => 'public']);
$id = DB::table('geo_locations')->where('external_id', $external)->value('id');
if ($level === 'province') {
$province = $id;
}
}
$batch = DB::table('demographic_imports')->where('sha256', $slice['sha256'])->value('id');
if (! $batch) {
$batch = (string) Str::uuid();
DB::table('demographic_imports')->insert(['id' => $batch, 'sha256' => $slice['sha256'], 'file_name' => $slice['file'], 'source' => 'Instituto Nacional de Estatística (INE)', 'publication_version' => 'Exemplar INE fornecido pela Tempo; edição não indicada; SHA256 identifica a versão', 'status' => 'approved', 'report' => json_encode(['issue_counts' => [], 'qa_slice' => true]), 'preview_sha256' => hash('sha256', json_encode($slice)), 'prepared_by' => $actor->id, 'created_at' => now(), 'updated_at' => now()]);
foreach ($slice['records'] as $row) {
unset($row['province'], $row['territory'], $row['level']);
DB::table('demographic_observations')->insert(['import_id' => $batch, 'geo_location_id' => $id, 'validation_status' => 'valid', ...$row]);
}
}
$result[$territoryName] = ['id' => $id, 'province' => $province, 'batch' => $batch, 'records' => count($slice['records'])];
}
return $result;
});
echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE).PHP_EOL;
+42
View File
@@ -0,0 +1,42 @@
"""Read-only D01-D06 QA slices and cell verification; never modifies INE workbooks."""
import argparse
import hashlib
import json
from pathlib import Path
from demography_workbook import parse
cli = argparse.ArgumentParser()
cli.add_argument('directory', type=Path)
cli.add_argument('--output', type=Path, required=True)
args = cli.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
slices, references = [], []
for name, territory, cell in [('Gaza.xlsx', 'Bilene', None),
('Inhambane.xlsx', 'Funhalouro', 'B54'),
('Maputo Provincia.xlsx', 'Moamba', 'B179')]:
original = args.directory / name
before = hashlib.sha256(original.read_bytes()).hexdigest()
parsed = parse(original)
rows = [r for r in parsed['records'] if r['territory'] == territory
and 2026 <= r['year'] <= 2028 and r['sex'] == 'all']
unchanged = before == hashlib.sha256(original.read_bytes()).hexdigest()
if not unchanged:
raise RuntimeError('Source changed during verification')
slices.append({'file': name, 'sha256': before, 'source_unchanged': unchanged, 'records': rows})
if cell:
matched = [r for r in rows if r['sheet'] == '2026' and r['cell'] == cell]
if len(matched) != 1:
raise RuntimeError('Reference not uniquely verified')
references.append({'file': name, 'sha256': before, 'source_unchanged': unchanged,
'state': 'verified_from_original_labels', 'observation': matched[0]})
initial = args.directory.parent / 'Inhambane.xlsx'
if initial.is_file():
before = hashlib.sha256(initial.read_bytes()).hexdigest()
matched = [r for r in parse(initial)['records'] if r['sheet'] == '2026' and r['cell'] == 'B54']
if len(matched) != 1 or before != hashlib.sha256(initial.read_bytes()).hexdigest():
raise RuntimeError('Initial Inhambane exemplar not verified')
references.append({'file': '../Inhambane.xlsx (exemplar inicial distinto)', 'sha256': before,
'source_unchanged': True, 'state': 'verified_from_original_labels', 'observation': matched[0]})
(args.output / 'source-slices.json').write_text(json.dumps(slices, ensure_ascii=False, indent=2), encoding='utf-8')
(args.output / 'source-references.json').write_text(json.dumps(references, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps({'files': len(slices), 'verified_cells': len(references), 'originals_unchanged': True}))