fix(demography): verify D01-D06 in isolated QA with INE traceability
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
// Read-only comparison of QA observations with exact INE slices.
|
||||
require __DIR__.'/continuity-qa-bootstrap.php';
|
||||
$app = require dirname(__DIR__).'/bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
$identity = DB::selectOne('select current_database() as name, inet_server_port() as port');
|
||||
throw_unless($identity->name === 'continuity_20261120_test' && (int) $identity->port === 55441, RuntimeException::class, 'Isolated QA only');
|
||||
$slices = json_decode(file_get_contents(dirname(__DIR__).'/outputs/demography-d01-d06-20261120/source-slices.json'), true, 512, JSON_THROW_ON_ERROR);
|
||||
$checked = 0;
|
||||
foreach ($slices as $slice) {
|
||||
$batch = DB::table('demographic_imports')->where('sha256', $slice['sha256'])->first();
|
||||
throw_unless($batch && $batch->file_name === $slice['file'], RuntimeException::class, 'Original provenance missing');
|
||||
foreach ($slice['records'] as $row) {
|
||||
$stored = DB::table('demographic_observations as o')->join('geo_locations as g', 'g.id', '=', 'o.geo_location_id')
|
||||
->where('o.import_id', $batch->id)->where('o.sheet', $row['sheet'])->where('o.cell', $row['cell'])->first(['o.*', 'g.name']);
|
||||
throw_unless($stored && $stored->name === $row['territory'] && $stored->numeric_value === $row['numeric_value'] && $stored->original_value === $row['original_value'], RuntimeException::class, 'Observation mismatch');
|
||||
$checked++;
|
||||
}
|
||||
}
|
||||
echo json_encode(['database' => $identity->name, 'port' => $identity->port, 'exact_observations_checked' => $checked, 'mismatches' => 0, 'writes' => false, 'traceability_preserved' => true], JSON_PRETTY_PRINT).PHP_EOL;
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
// QA fixture: exact read-only INE slices, synthetic territorial identities and no opportunities.
|
||||
require __DIR__.'/continuity-qa-bootstrap.php';
|
||||
$app = require dirname(__DIR__).'/bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
$identity = DB::selectOne('select current_database() as name, inet_server_port() as port');
|
||||
throw_unless($identity->name === 'continuity_20261120_test' && (int) $identity->port === 55441, RuntimeException::class, 'Isolated QA only');
|
||||
$slices = json_decode(file_get_contents(dirname(__DIR__).'/outputs/demography-d01-d06-20261120/source-slices.json'), true, 512, JSON_THROW_ON_ERROR);
|
||||
$result = DB::transaction(function () use ($slices) {
|
||||
$actor = DB::table('users')->first();
|
||||
throw_unless($actor, RuntimeException::class, 'Existing synthetic investor required');
|
||||
foreach (['comercio-e-servicos' => 'Comércio e serviços', 'turismo' => 'Turismo'] as $slug => $name) {
|
||||
DB::table('sectors')->insertOrIgnore(['slug' => $slug, 'name' => $name, 'is_active' => true, 'published_at' => now(), 'created_at' => now(), 'updated_at' => now()]);
|
||||
}
|
||||
$country = DB::table('geo_locations')->where('location_type', 'country')->value('id');
|
||||
$result = [];
|
||||
foreach ($slices as $slice) {
|
||||
$provinceName = $slice['records'][0]['province'];
|
||||
$territoryName = $slice['records'][0]['territory'];
|
||||
foreach ([['province', $provinceName], ['district', $territoryName]] as [$level, $name]) {
|
||||
$id = (string) Str::uuid();
|
||||
$external = '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff:D01-D06:'.$name;
|
||||
DB::table('geo_locations')->insertOrIgnore(['id' => $id, 'parent_id' => $level === 'province' ? $country : $province, 'name' => $name, 'code' => 'QA-D01-'.$name, 'location_type' => $level, 'source_system' => 'cartografia-workbook', 'external_id' => $external, 'data_classification' => 'public']);
|
||||
$id = DB::table('geo_locations')->where('external_id', $external)->value('id');
|
||||
if ($level === 'province') {
|
||||
$province = $id;
|
||||
}
|
||||
}
|
||||
$batch = DB::table('demographic_imports')->where('sha256', $slice['sha256'])->value('id');
|
||||
if (! $batch) {
|
||||
$batch = (string) Str::uuid();
|
||||
DB::table('demographic_imports')->insert(['id' => $batch, 'sha256' => $slice['sha256'], 'file_name' => $slice['file'], 'source' => 'Instituto Nacional de Estatística (INE)', 'publication_version' => 'Exemplar INE fornecido pela Tempo; edição não indicada; SHA256 identifica a versão', 'status' => 'approved', 'report' => json_encode(['issue_counts' => [], 'qa_slice' => true]), 'preview_sha256' => hash('sha256', json_encode($slice)), 'prepared_by' => $actor->id, 'created_at' => now(), 'updated_at' => now()]);
|
||||
foreach ($slice['records'] as $row) {
|
||||
unset($row['province'], $row['territory'], $row['level']);
|
||||
DB::table('demographic_observations')->insert(['import_id' => $batch, 'geo_location_id' => $id, 'validation_status' => 'valid', ...$row]);
|
||||
}
|
||||
}
|
||||
$result[$territoryName] = ['id' => $id, 'province' => $province, 'batch' => $batch, 'records' => count($slice['records'])];
|
||||
}
|
||||
|
||||
return $result;
|
||||
});
|
||||
echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE).PHP_EOL;
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Read-only D01-D06 QA slices and cell verification; never modifies INE workbooks."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from demography_workbook import parse
|
||||
|
||||
cli = argparse.ArgumentParser()
|
||||
cli.add_argument('directory', type=Path)
|
||||
cli.add_argument('--output', type=Path, required=True)
|
||||
args = cli.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
slices, references = [], []
|
||||
for name, territory, cell in [('Gaza.xlsx', 'Bilene', None),
|
||||
('Inhambane.xlsx', 'Funhalouro', 'B54'),
|
||||
('Maputo Provincia.xlsx', 'Moamba', 'B179')]:
|
||||
original = args.directory / name
|
||||
before = hashlib.sha256(original.read_bytes()).hexdigest()
|
||||
parsed = parse(original)
|
||||
rows = [r for r in parsed['records'] if r['territory'] == territory
|
||||
and 2026 <= r['year'] <= 2028 and r['sex'] == 'all']
|
||||
unchanged = before == hashlib.sha256(original.read_bytes()).hexdigest()
|
||||
if not unchanged:
|
||||
raise RuntimeError('Source changed during verification')
|
||||
slices.append({'file': name, 'sha256': before, 'source_unchanged': unchanged, 'records': rows})
|
||||
if cell:
|
||||
matched = [r for r in rows if r['sheet'] == '2026' and r['cell'] == cell]
|
||||
if len(matched) != 1:
|
||||
raise RuntimeError('Reference not uniquely verified')
|
||||
references.append({'file': name, 'sha256': before, 'source_unchanged': unchanged,
|
||||
'state': 'verified_from_original_labels', 'observation': matched[0]})
|
||||
initial = args.directory.parent / 'Inhambane.xlsx'
|
||||
if initial.is_file():
|
||||
before = hashlib.sha256(initial.read_bytes()).hexdigest()
|
||||
matched = [r for r in parse(initial)['records'] if r['sheet'] == '2026' and r['cell'] == 'B54']
|
||||
if len(matched) != 1 or before != hashlib.sha256(initial.read_bytes()).hexdigest():
|
||||
raise RuntimeError('Initial Inhambane exemplar not verified')
|
||||
references.append({'file': '../Inhambane.xlsx (exemplar inicial distinto)', 'sha256': before,
|
||||
'source_unchanged': True, 'state': 'verified_from_original_labels', 'observation': matched[0]})
|
||||
(args.output / 'source-slices.json').write_text(json.dumps(slices, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
(args.output / 'source-references.json').write_text(json.dumps(references, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps({'files': len(slices), 'verified_cells': len(references), 'originals_unchanged': True}))
|
||||
Reference in New Issue
Block a user