Deliver additive INE demography and authenticated homologation installation
tests / ci (push) Canceled after 0s

This commit is contained in:
2026-11-13 13:27:36 +02:00
parent 880e12890c
commit b20068d04b
31 changed files with 2971 additions and 2 deletions
+209
View File
@@ -0,0 +1,209 @@
<?php
namespace App\Services;
use App\Models\User;
use App\Services\Malware\MalwareScanner;
use App\Support\CheckedFile;
use App\Support\IsolatedQaDatabase;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Symfony\Component\Process\Process;
final class DemographicImport
{
public function authorize(User $actor): void
{
abort_unless($actor->can('manage-portal') && ! app(PortalProfileAccess::class)->has($actor, 'auditor'), 403);
abort_unless($actor->two_factor_confirmed_at && $actor->two_factor_secret, 403, 'Confirme MFA para gerir dados demográficos.');
if (! IsolatedQaDatabase::allowsSchemaChanges()) {
abort_unless(app(DemographicInstallation::class)->activeTarget() && app(DemographicEvidence::class)->installed(), 403, 'Destino demográfico não instalado ou não autorizado.');
abort_unless(app(PortalProfileAccess::class)->mfa($actor), 403, 'Conclua MFA nesta sessão para gerir dados demográficos.');
}
}
/** @return array<string,mixed> */
public function prepare(User $actor, string $file, string $name, string $version, ?int $baselineYear): array
{
$this->authorize($actor);
abort_unless(is_file($file) && filesize($file) <= 20_000_000 && strtolower(pathinfo($name, PATHINFO_EXTENSION)) === 'xlsx', 422);
$sha = CheckedFile::sha256($file);
$existing = DB::table('demographic_imports')->where('sha256', $sha)->first();
if ($existing) {
return $this->preview($actor, $existing->id);
}
$id = (string) Str::uuid();
$directory = 'demography/'.$id;
$disk = Storage::disk('local');
$disk->makeDirectory($directory);
$path = $disk->path($directory.'/original.xlsx');
abort_unless(copy($file, $path) && CheckedFile::matches($path, $sha), 409);
$scan = app(MalwareScanner::class)->scan($path, (int) filesize($path), $disk->path($directory));
abort_unless($scan['result'] === 'clean', 423, 'Análise antimalware pendente ou não limpa; original privado conservado.');
$output = $disk->path($directory.'/preview.json');
$command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')];
if ($baselineYear !== null) {
array_push($command, '--baseline-year', (string) $baselineYear);
}
$process = new Process($command, base_path(), timeout: 120);
$process->run();
abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.');
$report = json_decode(CheckedFile::read($output), true, 512, JSON_THROW_ON_ERROR);
abort_unless(hash_equals($sha, $report['sha256']) && CheckedFile::matches($path, $sha), 409);
$summary = array_diff_key($report, ['records' => true]);
$summary['scan'] = ['result' => $scan['result'], 'engine' => $scan['engine'], 'engine_version' => $scan['engine_version'], 'sha256' => $sha];
// The supplied publication label is provenance metadata, never an invented edition/date.
DB::table('demographic_imports')->insert(['id' => $id, 'sha256' => $sha, 'file_name' => basename($name),
'source' => $report['source'], 'publication_version' => $version, 'status' => 'prepared',
'report' => json_encode($summary, JSON_THROW_ON_ERROR), 'preview_sha256' => CheckedFile::sha256($output),
'prepared_by' => $actor->id, 'created_at' => now(), 'updated_at' => now()]);
$this->event($id, $actor, 'prepared', 'Pré-visualização do original; sem aprovação de utilização.', ['sha256' => $sha]);
return $this->preview($actor, $id);
}
public static function key(string $province, string $territory, string $level): string
{
return $province.'|'.$territory.'|'.$level;
}
/** @return array<string,mixed> */
public function preview(User $actor, string $id): array
{
$this->authorize($actor);
$batch = DB::table('demographic_imports')->where('id', $id)->first();
abort_unless($batch !== null, 404);
$report = json_decode($batch->report, true, 512, JSON_THROW_ON_ERROR);
$locations = DB::table('geo_locations')->whereIn('location_type', ['province', 'district'])->whereNull('deleted_at')->get(['id', 'name', 'parent_id', 'location_type']);
$matches = [];
foreach ($report['territories'] as [$province, $territory, $level]) {
$provinces = $locations->filter(fn ($g) => $g->location_type === 'province' && $this->normalize($g->name) === $this->normalize($province));
$candidates = $level === 'province' ? $provinces : $locations->filter(fn ($g) => $g->location_type === 'district' && $provinces->contains('id', $g->parent_id) && $this->normalize($g->name) === $this->normalize($territory));
$key = self::key($province, $territory, $level);
if ($level === 'district') {
$headers = $report['territory_headers'][$key] ?? [];
if ($headers !== [] && $candidates->count() > 1 && array_all($headers, fn ($h) => preg_match('/^Distrito de /i', $h) === 1)) {
$candidates = $candidates->filter(fn ($g) => ! preg_match('/^CIDADE /i', $g->name));
}
if ($headers !== [] && $candidates->count() > 1 && array_all($headers, fn ($h) => preg_match('/^Cidade (de|da) /i', $h) === 1)) {
$candidates = $candidates->filter(fn ($g) => preg_match('/^CIDADE /i', $g->name) === 1);
}
$manifest = DemographicInstallation::remainingManifest();
$hashes = array_column($manifest['files'], 'sha256');
$alias = in_array($batch->sha256, $hashes, true) ? ($manifest['aliases'][$key] ?? null) : null;
if ($alias !== null) {
$candidates = $locations->filter(fn ($g) => $g->location_type === 'district' && $provinces->contains('id', $g->parent_id) && $this->literal($g->name) === $this->literal($alias));
}
}
$matches[] = ['key' => self::key($province, $territory, $level), 'province' => $province, 'territory' => $territory, 'level' => $level,
'geo_location_id' => $candidates->count() === 1 ? $candidates->first()->id : null, 'state' => $candidates->count() === 1 ? 'matched' : ($candidates->isEmpty() ? 'missing' : 'ambiguous')];
}
return ['id' => $id, 'file_name' => $batch->file_name, 'sha256' => $batch->sha256, 'publication_version' => $batch->publication_version,
'status' => $batch->status, 'report' => $report, 'matches' => $matches,
'territory_options' => $locations->values()->all(), 'events' => DB::table('demographic_events')->where('import_id', $id)->orderBy('id')->get()];
}
private function normalize(string $name): string
{
return preg_replace('/^(cidade|distrito) (de|da) /', '', $this->literal($name));
}
private function literal(string $name): string
{
$name = str_replace(['`', '’', '‘', '´'], "'", Str::ascii($name));
return preg_replace('/\s+/', ' ', trim(strtolower(str_replace(['-', '–'], ' ', $name))));
}
/** @param array<string,string> $mapping
* @return array<string,mixed> */
public function confirm(User $actor, string $id, string $sha, array $mapping, string $reason): array
{
$this->authorize($actor);
return DB::transaction(function () use ($actor, $id, $sha, $mapping, $reason) {
$batch = DB::table('demographic_imports')->where('id', $id)->lockForUpdate()->first();
abort_unless($batch && hash_equals($batch->sha256, $sha), 409);
if ($batch->status !== 'prepared') {
return ['id' => $id, 'status' => $batch->status, 'records' => DB::table('demographic_observations')->where('import_id', $id)->count(), 'repeated' => true];
}
$disk = Storage::disk('local');
$file = $disk->path('demography/'.$id.'/preview.json');
abort_unless(is_file($file) && CheckedFile::matches($file, $batch->preview_sha256) && CheckedFile::matches($disk->path('demography/'.$id.'/original.xlsx'), $sha), 409, 'Prévia ou original alterados.');
$report = json_decode(CheckedFile::read($file), true, 512, JSON_THROW_ON_ERROR);
$blocking = array_intersect(array_keys($report['issue_counts']), ['conflicting_value', 'duplicate_value', 'unsupported_sheet', 'unsupported_population_headers', 'reference_year_required', 'title_year_mismatch', 'invalid_numeric_or_formula']);
abort_if($blocking !== [], 422, 'Corrija os erros estruturais antes de confirmar: '.implode(', ', $blocking));
$plan = $this->preview($actor, $id);
$resolved = [];
foreach ($plan['matches'] as $match) {
$locationId = $mapping[$match['key']] ?? $match['geo_location_id'];
$location = DB::table('geo_locations')->where('id', $locationId)->whereNull('deleted_at')->first();
abort_unless($location && $location->location_type === $match['level'], 422, 'Correspondência territorial por resolver.');
if ($match['level'] === 'district') {
$provinceKey = self::key($match['province'], $match['province'], 'province');
$provinceMatch = array_values(array_filter($plan['matches'], fn ($m) => $m['key'] === $provinceKey))[0] ?? null;
abort_unless($location->parent_id === ($mapping[$provinceKey] ?? $provinceMatch['geo_location_id'] ?? null), 422, 'Distrito fora da província indicada.');
}
abort_if(in_array($locationId, $resolved, true), 422, 'Duas designações não podem ser fundidas na mesma localização.');
$resolved[$match['key']] = $locationId;
}
$chunk = [];
$recordCount = 0;
foreach ($this->recordRows($report, $disk->path('demography/'.$id.'/records.ndjson')) as $row) {
$recordCount++;
$chunk[] = ['import_id' => $id, 'geo_location_id' => $resolved[self::key($row['province'], $row['territory'], $row['level'])],
...array_intersect_key($row, array_flip(['year', 'indicator', 'sex', 'residence', 'age_group', 'unit', 'data_kind', 'original_value', 'numeric_value', 'sheet', 'cell'])),
'validation_status' => $report['issues'] === [] ? 'valid' : 'warning'];
if (count($chunk) === 500) {
DB::table('demographic_observations')->insert($chunk);
$chunk = [];
}
}
if ($chunk !== []) {
DB::table('demographic_observations')->insert($chunk);
}
abort_unless($recordCount === $report['record_count'], 409, 'O número de observações difere da prévia.');
DB::table('demographic_imports')->where('id', $id)->update(['status' => 'imported', 'updated_at' => now()]);
$this->event($id, $actor, 'imported', $reason, ['mapping' => $resolved, 'records' => $recordCount, 'sha256' => $sha]);
return ['id' => $id, 'status' => 'imported', 'records' => $recordCount, 'repeated' => false];
});
}
/** @param array<string,mixed> $report
* @return iterable<array<string,mixed>> */
private function recordRows(array $report, string $path): iterable
{
if (! isset($report['records_file_sha256'])) {
yield from $report['records'];
return;
}
abort_unless(is_file($path) && CheckedFile::matches($path, $report['records_file_sha256']), 409, 'As observações diferem da prévia conferida.');
$stream = new \SplFileObject($path, 'r');
while (! $stream->eof()) {
$line = $stream->fgets();
if (trim($line) === '') { continue; }
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
}
}
public function transition(User $actor, string $id, string $action, string $reason): void
{
$this->authorize($actor);
DB::transaction(function () use ($actor, $id, $action, $reason) {
$batch = DB::table('demographic_imports')->where('id', $id)->lockForUpdate()->first();
abort_unless($batch !== null, 404);
abort_unless(in_array($action, ['approved', 'disabled'], true) && $batch->status !== 'prepared', 422);
// No deletes, replacement of other editions, or automatic publication in production.
DB::table('demographic_imports')->where('id', $id)->update(['status' => $action, 'updated_at' => now()]);
$this->event($id, $actor, $action, $reason, ['previous_status' => $batch->status]);
});
}
/** @param array<string,mixed> $details */
private function event(string $id, User $actor, string $action, string $reason, array $details): void
{
DB::table('demographic_events')->insert(['import_id' => $id, 'actor_id' => $actor->id, 'action' => $action, 'reason' => $reason, 'details' => json_encode($details, JSON_THROW_ON_ERROR), 'created_at' => now()]);
}
}