From 4988ca89dc6dddbde4efdada24546767ffcdd411 Mon Sep 17 00:00:00 2001 From: nala Date: Fri, 13 Nov 2026 13:57:01 +0200 Subject: [PATCH] Use verified CLI preparations when web PHP cannot execute processes --- app/Services/DemographicImport.php | 62 +++++++++++++++++-- app/Services/DemographicInstallation.php | 10 ++- config/demography.php | 1 + tests/Feature/DemographicInstallationTest.php | 9 +++ tests/Feature/DemographyTest.php | 35 +++++++++++ tools/demography_workbook.py | 9 +++ 6 files changed, 117 insertions(+), 9 deletions(-) diff --git a/app/Services/DemographicImport.php b/app/Services/DemographicImport.php index 35ad4cd5..6121ade5 100644 --- a/app/Services/DemographicImport.php +++ b/app/Services/DemographicImport.php @@ -13,6 +13,48 @@ use Symfony\Component\Process\Process; final class DemographicImport { + /** + * CLI preparation is read-only input, never an import or approval. + * @return array{preview:string,records:string,receipt:array}|null + */ + public function preparedSource(string $sha, ?int $baselineYear, bool $verifyRecords = true): ?array + { + $root = config('demography.prepared_directory'); + if (! is_string($root) || $root === '') { return null; } + abort_unless(preg_match('/^[a-f0-9]{64}$/', $sha) === 1, 422); + $directory = $root.'/'.$sha; + if (! is_file($directory.'/preparation.json')) { return null; } + abort_unless(is_readable($directory.'/preparation.json'), 409, 'Recibo CLI indisponível.'); + try { $receipt = json_decode(CheckedFile::read($directory.'/preparation.json'), true, 512, JSON_THROW_ON_ERROR); } + catch (\JsonException) { abort(409, 'Recibo CLI inválido.'); } + abort_unless(is_array($receipt), 409, 'Recibo CLI inválido.'); + $preview = $directory.'/preview.json'; + $records = $directory.'/records.ndjson'; + abort_unless(($receipt['sha256'] ?? '') === $sha && array_key_exists('baseline_year', $receipt) && $receipt['baseline_year'] === $baselineYear + && ($receipt['parser_sha256'] ?? '') === CheckedFile::sha256(base_path('tools/demography_workbook.py')) + && is_file($preview) && CheckedFile::matches($preview, $receipt['preview_sha256'] ?? '') && is_file($records), 409, 'A preparação CLI mudou; conservar o original e preparar novamente fora do servidor web.'); + try { $report = json_decode(CheckedFile::read($preview), true, 512, JSON_THROW_ON_ERROR); } + catch (\JsonException) { abort(409, 'Prévia CLI inválida.'); } + abort_unless(is_array($report), 409, 'Prévia CLI inválida.'); + abort_unless(($report['sha256'] ?? '') === $sha && ($report['record_count'] ?? null) === ($receipt['record_count'] ?? null) + && ($report['records_file_sha256'] ?? '') === ($receipt['records_sha256'] ?? '') + && (! $verifyRecords || CheckedFile::matches($records, $receipt['records_sha256'] ?? '')), 409, 'Resultados da preparação CLI alterados.'); + + return ['preview' => $preview, 'records' => $records, 'receipt' => $receipt]; + } + + /** @param array> $files */ + public function preparedSourcesReady(array $files): bool + { + try { + foreach ($files as $name => $expected) { + $source = $this->preparedSource($expected['sha256'], $name === 'Dados demograficos.xlsx' ? 2017 : null, false); + if ($source === null || $source['receipt']['record_count'] !== $expected['records']) { return false; } + } + return $files !== []; + } catch (\Throwable) { return false; } + } + public function authorize(User $actor): void { $access = app(PortalProfileAccess::class); @@ -43,13 +85,21 @@ final class DemographicImport $scan = app(MalwareScanner::class)->scan($path, (int) filesize($path), $disk->path($directory)); abort_unless($scan['result'] === 'clean', 423, 'Análise antimalware pendente ou não limpa; original privado conservado.'); $output = $disk->path($directory.'/preview.json'); - $command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')]; - if ($baselineYear !== null) { - array_push($command, '--baseline-year', (string) $baselineYear); + $prepared = $this->preparedSource($sha, $baselineYear); + if ($prepared !== null) { + abort_unless(copy($prepared['preview'], $output) && CheckedFile::matches($output, $prepared['receipt']['preview_sha256']) + && copy($prepared['records'], $disk->path($directory.'/records.ndjson')) + && CheckedFile::matches($disk->path($directory.'/records.ndjson'), $prepared['receipt']['records_sha256']), 409, 'Preparação CLI alterada durante a cópia.'); + } else { + abort_unless(function_exists('proc_open'), 503, 'Este servidor web não executa o preparador. Prepare este original pelo importador CLI e confira o respectivo recibo antes de continuar.'); + $command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')]; + if ($baselineYear !== null) { + array_push($command, '--baseline-year', (string) $baselineYear); + } + $process = new Process($command, base_path(), timeout: 120); + $process->run(); + abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.'); } - $process = new Process($command, base_path(), timeout: 120); - $process->run(); - abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.'); $report = json_decode(CheckedFile::read($output), true, 512, JSON_THROW_ON_ERROR); abort_unless(hash_equals($sha, $report['sha256']) && CheckedFile::matches($path, $sha), 409); $summary = array_diff_key($report, ['records' => true]); diff --git a/app/Services/DemographicInstallation.php b/app/Services/DemographicInstallation.php index ada3740a..9555ca88 100644 --- a/app/Services/DemographicInstallation.php +++ b/app/Services/DemographicInstallation.php @@ -136,10 +136,14 @@ final class DemographicInstallation $tables = array_map(fn ($t) => Schema::hasTable($t), ['demographic_imports', 'demographic_observations', 'demographic_events']); $ledger = DB::table('migrations')->where('migration', self::MIGRATION)->exists(); $consistent = ($ledger && ! in_array(false, $tables, true)) || (! $ledger && ! in_array(true, $tables, true)); - $runtime = new Process([config('demography.python', 'python'), '-c', 'import sys; assert sys.version_info >= (3, 9)'], base_path(), timeout: 15); + $preparedFiles = self::FILES + self::remainingManifest()['files'] + self::zambeziaManifest()['files']; + $runtimeReady = app(DemographicImport::class)->preparedSourcesReady($preparedFiles); try { - $runtime->run(); - $runtimeReady = $runtime->isSuccessful(); + if (! $runtimeReady && function_exists('proc_open')) { + $runtime = new Process([config('demography.python', 'python'), '-c', 'import sys; assert sys.version_info >= (3, 9)'], base_path(), timeout: 15); + $runtime->run(); + $runtimeReady = $runtime->isSuccessful(); + } } catch (ExceptionInterface) { $runtimeReady = false; } diff --git a/config/demography.php b/config/demography.php index 7233d3b6..faa0b26e 100644 --- a/config/demography.php +++ b/config/demography.php @@ -5,6 +5,7 @@ return [ 'enabled' => env('DEMOGRAPHY_ENABLED', env('APP_URL') !== 'https://teste-investidor.tempo.co.mz' && in_array(env('APP_ENV', 'production'), ['local', 'testing'], true)), 'source_directory' => env('DEMOGRAPHY_SOURCE_DIRECTORY', 'C:/Mac/Home/Documents/TEMPO/Portal Tempo/Servicos Tempo/portal do investidor/demografia/Provincias'), + 'prepared_directory' => env('DEMOGRAPHY_PREPARED_DIRECTORY'), // An explicit interpreter avoids Windows Store execution aliases in web workers. 'python' => env('DEMOGRAPHY_PYTHON', 'python'), ]; diff --git a/tests/Feature/DemographicInstallationTest.php b/tests/Feature/DemographicInstallationTest.php index 538d49ba..3018e22e 100644 --- a/tests/Feature/DemographicInstallationTest.php +++ b/tests/Feature/DemographicInstallationTest.php @@ -64,6 +64,15 @@ final class DemographicInstallationTest extends TestCase $this->assertStringContainsString('interpretador', $plan['notice']); } + public function test_disabled_process_support_does_not_crash_the_installation_page(): void + { + if (function_exists('proc_open')) { $this->markTestSkipped('Run this regression with -d disable_functions=proc_open.'); } + config(['demography.prepared_directory' => null]); + $this->actingAs($this->manager)->withSession(['portal.mfa_actor' => $this->manager->id, 'portal.mfa_at' => time()]); + $this->get('/management/demography/install')->assertOk()->assertInertia(fn (\Inertia\Testing\AssertableInertia $page) => $page + ->component('management/demography-install')->where('plan.runtime_ready', false)->where('plan.ready', false)); + } + public function test_schema_flag_alone_does_not_authorise_the_operational_target(): void { $this->app['env'] = 'local'; diff --git a/tests/Feature/DemographyTest.php b/tests/Feature/DemographyTest.php index 2a1d3903..ad0c1b8d 100644 --- a/tests/Feature/DemographyTest.php +++ b/tests/Feature/DemographyTest.php @@ -198,6 +198,41 @@ final class DemographyTest extends TestCase $migration->down(); } + public function test_cli_preparation_imports_without_process_execution_and_keeps_exact_values(): void + { + Storage::fake('local'); + $disk = Storage::disk('local'); + $original = 'SYNTHETIC prepared original'; $sha = hash('sha256', $original); + $directory = 'prepared/'.$sha; $disk->makeDirectory($directory); + $disk->put('synthetic.xlsx', $original); + $row = ['province' => 'SYNTHETIC province', 'territory' => 'SYNTHETIC province', 'level' => 'province', + 'year' => 2026, 'indicator' => 'population', 'sex' => 'all', 'residence' => 'all', 'age_group' => 'all', 'unit' => 'pessoas', + 'data_kind' => 'projected', 'original_value' => '100.125', 'numeric_value' => '100.125', 'sheet' => '2026', 'cell' => 'B4']; + $records = json_encode($row, JSON_THROW_ON_ERROR)."\n"; + $report = ['sha256' => $sha, 'source' => 'SYNTHETIC QA ONLY', 'record_count' => 1, 'records_file_sha256' => hash('sha256', $records), + 'territories' => [['SYNTHETIC province', 'SYNTHETIC province', 'province']], 'issues' => [], 'issue_counts' => []]; + $preview = json_encode($report, JSON_THROW_ON_ERROR); + $receipt = ['sha256' => $sha, 'baseline_year' => null, 'parser_sha256' => hash_file('sha256', base_path('tools/demography_workbook.py')), + 'preview_sha256' => hash('sha256', $preview), 'records_sha256' => hash('sha256', $records), 'record_count' => 1]; + $disk->put($directory.'/preview.json', $preview); $disk->put($directory.'/records.ndjson', $records); + $disk->put($directory.'/preparation.json', json_encode($receipt, JSON_THROW_ON_ERROR)); + config(['demography.prepared_directory' => $disk->path('prepared'), 'demography.python' => '/missing-python-must-not-run']); + $this->mock(\App\Services\Malware\MalwareScanner::class)->shouldReceive('scan')->once()->andReturn([ + 'result' => 'clean', 'engine' => 'synthetic', 'engine_version' => 'test']); + $service = app(DemographicImport::class); + $this->assertTrue($service->preparedSourcesReady(['synthetic.xlsx' => ['sha256' => $sha, 'records' => 1]])); + $result = $service->prepare($this->manager, $disk->path('synthetic.xlsx'), 'synthetic.xlsx', 'SYNTHETIC', null); + $this->assertSame(0, DB::table('demographic_observations')->where('import_id', $result['id'])->count()); + $service->confirm($this->manager, $result['id'], $sha, [], 'Synthetic authenticated import from CLI preparation.'); + $this->assertSame('100.125', DB::table('demographic_observations')->where('import_id', $result['id'])->value('numeric_value')); + $this->assertSame('imported', DB::table('demographic_imports')->where('id', $result['id'])->value('status')); + $disk->put($directory.'/records.ndjson', 'TAMPERED'); + try { $service->preparedSource($sha, null); $this->fail('Changed prepared records must be refused.'); } + catch (\Symfony\Component\HttpKernel\Exception\HttpException $error) { $this->assertSame(409, $error->getStatusCode()); } + $receipt['parser_sha256'] = str_repeat('0', 64); $disk->put($directory.'/preparation.json', json_encode($receipt, JSON_THROW_ON_ERROR)); + $this->assertFalse($service->preparedSourcesReady(['synthetic.xlsx' => ['sha256' => $sha, 'records' => 1]])); + } + public function test_remote_consultation_requires_explicit_activation_and_preserves_versions(): void { $environment = app()->environment(); diff --git a/tools/demography_workbook.py b/tools/demography_workbook.py index 827461f8..748c79e4 100644 --- a/tools/demography_workbook.py +++ b/tools/demography_workbook.py @@ -267,7 +267,10 @@ if __name__ == "__main__": cli.add_argument("--baseline-year", type=int) cli.add_argument("--output", required=True) cli.add_argument("--records-output") + cli.add_argument("--preparation-receipt") args = cli.parse_args() + if args.preparation_receipt and not args.records_output: + cli.error('--preparation-receipt requires --records-output') result = parse(args.file, args.baseline_year) if args.records_output: record_path = Path(args.records_output) @@ -276,4 +279,10 @@ if __name__ == "__main__": target.write(json.dumps(record, ensure_ascii=False, separators=(',', ':'))+'\n') result['records_file_sha256'] = hashlib.sha256(record_path.read_bytes()).hexdigest() Path(args.output).write_text(json.dumps(result, ensure_ascii=False, separators=(",", ":")), encoding="utf-8") + if args.preparation_receipt: + receipt = dict(sha256=result['sha256'], baseline_year=args.baseline_year, + parser_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + preview_sha256=hashlib.sha256(Path(args.output).read_bytes()).hexdigest(), + records_sha256=result['records_file_sha256'], record_count=result['record_count']) + Path(args.preparation_receipt).write_text(json.dumps(receipt, separators=(',', ':')), encoding='utf-8') print(json.dumps({k: v for k, v in result.items() if k not in ("records", "issues")}, ensure_ascii=False))