261 lines
17 KiB
PHP
261 lines
17 KiB
PHP
<?php
|
||
|
||
namespace App\Services;
|
||
|
||
use App\Models\User;
|
||
use App\Services\Malware\MalwareScanner;
|
||
use App\Support\CheckedFile;
|
||
use App\Support\IsolatedQaDatabase;
|
||
use Illuminate\Support\Facades\DB;
|
||
use Illuminate\Support\Facades\Storage;
|
||
use Illuminate\Support\Str;
|
||
use Symfony\Component\Process\Process;
|
||
|
||
final class DemographicImport
|
||
{
|
||
/**
|
||
* CLI preparation is read-only input, never an import or approval.
|
||
* @return array{preview:string,records:string,receipt:array<string,mixed>}|null
|
||
*/
|
||
public function preparedSource(string $sha, ?int $baselineYear, bool $verifyRecords = true): ?array
|
||
{
|
||
$root = config('demography.prepared_directory');
|
||
if (! is_string($root) || $root === '') { return null; }
|
||
abort_unless(preg_match('/^[a-f0-9]{64}$/', $sha) === 1, 422);
|
||
$directory = $root.'/'.$sha;
|
||
if (! is_file($directory.'/preparation.json')) { return null; }
|
||
abort_unless(is_readable($directory.'/preparation.json'), 409, 'Recibo CLI indisponível.');
|
||
try { $receipt = json_decode(CheckedFile::read($directory.'/preparation.json'), true, 512, JSON_THROW_ON_ERROR); }
|
||
catch (\JsonException) { abort(409, 'Recibo CLI inválido.'); }
|
||
abort_unless(is_array($receipt), 409, 'Recibo CLI inválido.');
|
||
$preview = $directory.'/preview.json';
|
||
$records = $directory.'/records.ndjson';
|
||
abort_unless(($receipt['sha256'] ?? '') === $sha && array_key_exists('baseline_year', $receipt) && $receipt['baseline_year'] === $baselineYear
|
||
&& ($receipt['parser_sha256'] ?? '') === CheckedFile::sha256(base_path('tools/demography_workbook.py'))
|
||
&& is_file($preview) && CheckedFile::matches($preview, $receipt['preview_sha256'] ?? '') && is_file($records), 409, 'A preparação CLI mudou; conservar o original e preparar novamente fora do servidor web.');
|
||
try { $report = json_decode(CheckedFile::read($preview), true, 512, JSON_THROW_ON_ERROR); }
|
||
catch (\JsonException) { abort(409, 'Prévia CLI inválida.'); }
|
||
abort_unless(is_array($report), 409, 'Prévia CLI inválida.');
|
||
abort_unless(($report['sha256'] ?? '') === $sha && ($report['record_count'] ?? null) === ($receipt['record_count'] ?? null)
|
||
&& ($report['records_file_sha256'] ?? '') === ($receipt['records_sha256'] ?? '')
|
||
&& (! $verifyRecords || CheckedFile::matches($records, $receipt['records_sha256'] ?? '')), 409, 'Resultados da preparação CLI alterados.');
|
||
|
||
return ['preview' => $preview, 'records' => $records, 'receipt' => $receipt];
|
||
}
|
||
|
||
/** @param array<string,array<string,mixed>> $files */
|
||
public function preparedSourcesReady(array $files): bool
|
||
{
|
||
try {
|
||
foreach ($files as $name => $expected) {
|
||
$source = $this->preparedSource($expected['sha256'], $name === 'Dados demograficos.xlsx' ? 2017 : null, false);
|
||
if ($source === null || $source['receipt']['record_count'] !== $expected['records']) { return false; }
|
||
}
|
||
return $files !== [];
|
||
} catch (\Throwable) { return false; }
|
||
}
|
||
|
||
public function authorize(User $actor): void
|
||
{
|
||
$access = app(PortalProfileAccess::class);
|
||
abort_unless(($actor->can('manage-portal') || $access->superuser($actor)) && ! $access->has($actor, 'auditor'), 403);
|
||
abort_unless($actor->two_factor_confirmed_at && $actor->two_factor_secret, 403, 'Confirme MFA para gerir dados demográficos.');
|
||
if (! IsolatedQaDatabase::allowsSchemaChanges()) {
|
||
abort_unless(app(DemographicInstallation::class)->activeTarget() && app(DemographicEvidence::class)->installed(), 403, 'Destino demográfico não instalado ou não autorizado.');
|
||
abort_unless(app(PortalProfileAccess::class)->mfa($actor), 403, 'Conclua MFA nesta sessão para gerir dados demográficos.');
|
||
}
|
||
}
|
||
|
||
/** @return array<string,mixed> */
|
||
public function prepare(User $actor, string $file, string $name, string $version, ?int $baselineYear): array
|
||
{
|
||
$this->authorize($actor);
|
||
abort_unless(is_file($file) && filesize($file) <= 20_000_000 && strtolower(pathinfo($name, PATHINFO_EXTENSION)) === 'xlsx', 422);
|
||
$sha = CheckedFile::sha256($file);
|
||
$existing = DB::table('demographic_imports')->where('sha256', $sha)->first();
|
||
if ($existing) {
|
||
return $this->preview($actor, $existing->id);
|
||
}
|
||
$id = (string) Str::uuid();
|
||
$directory = 'demography/'.$id;
|
||
$disk = Storage::disk('local');
|
||
$disk->makeDirectory($directory);
|
||
$path = $disk->path($directory.'/original.xlsx');
|
||
abort_unless(copy($file, $path) && CheckedFile::matches($path, $sha), 409);
|
||
$scan = app(MalwareScanner::class)->scan($path, (int) filesize($path), $disk->path($directory));
|
||
abort_unless($scan['result'] === 'clean', 423, 'Análise antimalware pendente ou não limpa; original privado conservado.');
|
||
$output = $disk->path($directory.'/preview.json');
|
||
$prepared = $this->preparedSource($sha, $baselineYear);
|
||
if ($prepared !== null) {
|
||
abort_unless(copy($prepared['preview'], $output) && CheckedFile::matches($output, $prepared['receipt']['preview_sha256'])
|
||
&& copy($prepared['records'], $disk->path($directory.'/records.ndjson'))
|
||
&& CheckedFile::matches($disk->path($directory.'/records.ndjson'), $prepared['receipt']['records_sha256']), 409, 'Preparação CLI alterada durante a cópia.');
|
||
} else {
|
||
abort_unless(function_exists('proc_open'), 503, 'Este servidor web não executa o preparador. Prepare este original pelo importador CLI e confira o respectivo recibo antes de continuar.');
|
||
$command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')];
|
||
if ($baselineYear !== null) {
|
||
array_push($command, '--baseline-year', (string) $baselineYear);
|
||
}
|
||
$process = new Process($command, base_path(), timeout: 120);
|
||
$process->run();
|
||
abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.');
|
||
}
|
||
$report = json_decode(CheckedFile::read($output), true, 512, JSON_THROW_ON_ERROR);
|
||
abort_unless(hash_equals($sha, $report['sha256']) && CheckedFile::matches($path, $sha), 409);
|
||
$summary = array_diff_key($report, ['records' => true]);
|
||
$summary['scan'] = ['result' => $scan['result'], 'engine' => $scan['engine'], 'engine_version' => $scan['engine_version'], 'sha256' => $sha];
|
||
// The supplied publication label is provenance metadata, never an invented edition/date.
|
||
DB::table('demographic_imports')->insert(['id' => $id, 'sha256' => $sha, 'file_name' => basename($name),
|
||
'source' => $report['source'], 'publication_version' => $version, 'status' => 'prepared',
|
||
'report' => json_encode($summary, JSON_THROW_ON_ERROR), 'preview_sha256' => CheckedFile::sha256($output),
|
||
'prepared_by' => $actor->id, 'created_at' => now(), 'updated_at' => now()]);
|
||
$this->event($id, $actor, 'prepared', 'Pré-visualização do original; sem aprovação de utilização.', ['sha256' => $sha]);
|
||
|
||
return $this->preview($actor, $id);
|
||
}
|
||
|
||
public static function key(string $province, string $territory, string $level): string
|
||
{
|
||
return $province.'|'.$territory.'|'.$level;
|
||
}
|
||
|
||
/** @return array<string,mixed> */
|
||
public function preview(User $actor, string $id): array
|
||
{
|
||
$this->authorize($actor);
|
||
$batch = DB::table('demographic_imports')->where('id', $id)->first();
|
||
abort_unless($batch !== null, 404);
|
||
$report = json_decode($batch->report, true, 512, JSON_THROW_ON_ERROR);
|
||
$locations = DB::table('geo_locations')->whereIn('location_type', ['province', 'district'])->whereNull('deleted_at')->get(['id', 'name', 'parent_id', 'location_type']);
|
||
$matches = [];
|
||
foreach ($report['territories'] as [$province, $territory, $level]) {
|
||
$provinces = $locations->filter(fn ($g) => $g->location_type === 'province' && $this->normalize($g->name) === $this->normalize($province));
|
||
$candidates = $level === 'province' ? $provinces : $locations->filter(fn ($g) => $g->location_type === 'district' && $provinces->contains('id', $g->parent_id) && $this->normalize($g->name) === $this->normalize($territory));
|
||
$key = self::key($province, $territory, $level);
|
||
if ($level === 'district') {
|
||
$headers = $report['territory_headers'][$key] ?? [];
|
||
if ($headers !== [] && $candidates->count() > 1 && array_all($headers, fn ($h) => preg_match('/^Distrito de /i', $h) === 1)) {
|
||
$candidates = $candidates->filter(fn ($g) => ! preg_match('/^CIDADE /i', $g->name));
|
||
}
|
||
if ($headers !== [] && $candidates->count() > 1 && array_all($headers, fn ($h) => preg_match('/^Cidade (de|da) /i', $h) === 1)) {
|
||
$candidates = $candidates->filter(fn ($g) => preg_match('/^CIDADE /i', $g->name) === 1);
|
||
}
|
||
$manifest = DemographicInstallation::remainingManifest();
|
||
$hashes = array_column($manifest['files'], 'sha256');
|
||
$alias = in_array($batch->sha256, $hashes, true) ? ($manifest['aliases'][$key] ?? null) : null;
|
||
if ($alias !== null) {
|
||
$candidates = $locations->filter(fn ($g) => $g->location_type === 'district' && $provinces->contains('id', $g->parent_id) && $this->literal($g->name) === $this->literal($alias));
|
||
}
|
||
}
|
||
$matches[] = ['key' => self::key($province, $territory, $level), 'province' => $province, 'territory' => $territory, 'level' => $level,
|
||
'geo_location_id' => $candidates->count() === 1 ? $candidates->first()->id : null, 'state' => $candidates->count() === 1 ? 'matched' : ($candidates->isEmpty() ? 'missing' : 'ambiguous')];
|
||
}
|
||
|
||
return ['id' => $id, 'file_name' => $batch->file_name, 'sha256' => $batch->sha256, 'publication_version' => $batch->publication_version,
|
||
'status' => $batch->status, 'report' => $report, 'matches' => $matches,
|
||
'territory_options' => $locations->values()->all(), 'events' => DB::table('demographic_events')->where('import_id', $id)->orderBy('id')->get()];
|
||
}
|
||
|
||
private function normalize(string $name): string
|
||
{
|
||
return preg_replace('/^(cidade|distrito) (de|da) /', '', $this->literal($name));
|
||
}
|
||
|
||
private function literal(string $name): string
|
||
{
|
||
$name = str_replace(['`', '’', '‘', '´'], "'", Str::ascii($name));
|
||
return preg_replace('/\s+/', ' ', trim(strtolower(str_replace(['-', '–'], ' ', $name))));
|
||
}
|
||
|
||
/** @param array<string,string> $mapping
|
||
* @return array<string,mixed> */
|
||
public function confirm(User $actor, string $id, string $sha, array $mapping, string $reason): array
|
||
{
|
||
$this->authorize($actor);
|
||
|
||
return DB::transaction(function () use ($actor, $id, $sha, $mapping, $reason) {
|
||
$batch = DB::table('demographic_imports')->where('id', $id)->lockForUpdate()->first();
|
||
abort_unless($batch && hash_equals($batch->sha256, $sha), 409);
|
||
if ($batch->status !== 'prepared') {
|
||
return ['id' => $id, 'status' => $batch->status, 'records' => DB::table('demographic_observations')->where('import_id', $id)->count(), 'repeated' => true];
|
||
}
|
||
$disk = Storage::disk('local');
|
||
$file = $disk->path('demography/'.$id.'/preview.json');
|
||
abort_unless(is_file($file) && CheckedFile::matches($file, $batch->preview_sha256) && CheckedFile::matches($disk->path('demography/'.$id.'/original.xlsx'), $sha), 409, 'Prévia ou original alterados.');
|
||
$report = json_decode(CheckedFile::read($file), true, 512, JSON_THROW_ON_ERROR);
|
||
$blocking = array_intersect(array_keys($report['issue_counts']), ['conflicting_value', 'duplicate_value', 'unsupported_sheet', 'unsupported_population_headers', 'reference_year_required', 'title_year_mismatch', 'invalid_numeric_or_formula']);
|
||
abort_if($blocking !== [], 422, 'Corrija os erros estruturais antes de confirmar: '.implode(', ', $blocking));
|
||
$plan = $this->preview($actor, $id);
|
||
$resolved = [];
|
||
foreach ($plan['matches'] as $match) {
|
||
$locationId = $mapping[$match['key']] ?? $match['geo_location_id'];
|
||
$location = DB::table('geo_locations')->where('id', $locationId)->whereNull('deleted_at')->first();
|
||
abort_unless($location && $location->location_type === $match['level'], 422, 'Correspondência territorial por resolver.');
|
||
if ($match['level'] === 'district') {
|
||
$provinceKey = self::key($match['province'], $match['province'], 'province');
|
||
$provinceMatch = array_values(array_filter($plan['matches'], fn ($m) => $m['key'] === $provinceKey))[0] ?? null;
|
||
abort_unless($location->parent_id === ($mapping[$provinceKey] ?? $provinceMatch['geo_location_id'] ?? null), 422, 'Distrito fora da província indicada.');
|
||
}
|
||
abort_if(in_array($locationId, $resolved, true), 422, 'Duas designações não podem ser fundidas na mesma localização.');
|
||
$resolved[$match['key']] = $locationId;
|
||
}
|
||
$chunk = [];
|
||
$recordCount = 0;
|
||
foreach ($this->recordRows($report, $disk->path('demography/'.$id.'/records.ndjson')) as $row) {
|
||
$recordCount++;
|
||
$chunk[] = ['import_id' => $id, 'geo_location_id' => $resolved[self::key($row['province'], $row['territory'], $row['level'])],
|
||
...array_intersect_key($row, array_flip(['year', 'indicator', 'sex', 'residence', 'age_group', 'unit', 'data_kind', 'original_value', 'numeric_value', 'sheet', 'cell'])),
|
||
'validation_status' => $report['issues'] === [] ? 'valid' : 'warning'];
|
||
if (count($chunk) === 500) {
|
||
DB::table('demographic_observations')->insert($chunk);
|
||
$chunk = [];
|
||
}
|
||
}
|
||
if ($chunk !== []) {
|
||
DB::table('demographic_observations')->insert($chunk);
|
||
}
|
||
abort_unless($recordCount === $report['record_count'], 409, 'O número de observações difere da prévia.');
|
||
DB::table('demographic_imports')->where('id', $id)->update(['status' => 'imported', 'updated_at' => now()]);
|
||
$this->event($id, $actor, 'imported', $reason, ['mapping' => $resolved, 'records' => $recordCount, 'sha256' => $sha]);
|
||
|
||
return ['id' => $id, 'status' => 'imported', 'records' => $recordCount, 'repeated' => false];
|
||
});
|
||
}
|
||
|
||
/** @param array<string,mixed> $report
|
||
* @return iterable<array<string,mixed>> */
|
||
private function recordRows(array $report, string $path): iterable
|
||
{
|
||
if (! isset($report['records_file_sha256'])) {
|
||
yield from $report['records'];
|
||
return;
|
||
}
|
||
abort_unless(is_file($path) && CheckedFile::matches($path, $report['records_file_sha256']), 409, 'As observações diferem da prévia conferida.');
|
||
$stream = new \SplFileObject($path, 'r');
|
||
while (! $stream->eof()) {
|
||
$line = $stream->fgets();
|
||
if (trim($line) === '') { continue; }
|
||
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||
}
|
||
}
|
||
|
||
public function transition(User $actor, string $id, string $action, string $reason): void
|
||
{
|
||
$this->authorize($actor);
|
||
DB::transaction(function () use ($actor, $id, $action, $reason) {
|
||
$batch = DB::table('demographic_imports')->where('id', $id)->lockForUpdate()->first();
|
||
abort_unless($batch !== null, 404);
|
||
abort_unless(in_array($action, ['approved', 'disabled'], true) && $batch->status !== 'prepared', 422);
|
||
// No deletes, replacement of other editions, or automatic publication in production.
|
||
DB::table('demographic_imports')->where('id', $id)->update(['status' => $action, 'updated_at' => now()]);
|
||
$this->event($id, $actor, $action, $reason, ['previous_status' => $batch->status]);
|
||
});
|
||
}
|
||
|
||
/** @param array<string,mixed> $details */
|
||
private function event(string $id, User $actor, string $action, string $reason, array $details): void
|
||
{
|
||
DB::table('demographic_events')->insert(['import_id' => $id, 'actor_id' => $actor->id, 'action' => $action, 'reason' => $reason, 'details' => json_encode($details, JSON_THROW_ON_ERROR), 'created_at' => now()]);
|
||
}
|
||
}
|