Use verified CLI preparations when web PHP cannot execute processes

app/Services/DemographicImport.php

@@ -13,6 +13,48 @@ use Symfony\Component\Process\Process;
 
 final class DemographicImport
 {
+    /**
+     * CLI preparation is read-only input, never an import or approval.
+     * @return array{preview:string,records:string,receipt:array<string,mixed>}|null
+     */
+    public function preparedSource(string $sha, ?int $baselineYear, bool $verifyRecords = true): ?array
+    {
+        $root = config('demography.prepared_directory');
+        if (! is_string($root) || $root === '') { return null; }
+        abort_unless(preg_match('/^[a-f0-9]{64}$/', $sha) === 1, 422);
+        $directory = $root.'/'.$sha;
+        if (! is_file($directory.'/preparation.json')) { return null; }
+        abort_unless(is_readable($directory.'/preparation.json'), 409, 'Recibo CLI indisponível.');
+        try { $receipt = json_decode(CheckedFile::read($directory.'/preparation.json'), true, 512, JSON_THROW_ON_ERROR); }
+        catch (\JsonException) { abort(409, 'Recibo CLI inválido.'); }
+        abort_unless(is_array($receipt), 409, 'Recibo CLI inválido.');
+        $preview = $directory.'/preview.json';
+        $records = $directory.'/records.ndjson';
+        abort_unless(($receipt['sha256'] ?? '') === $sha && array_key_exists('baseline_year', $receipt) && $receipt['baseline_year'] === $baselineYear
+            && ($receipt['parser_sha256'] ?? '') === CheckedFile::sha256(base_path('tools/demography_workbook.py'))
+            && is_file($preview) && CheckedFile::matches($preview, $receipt['preview_sha256'] ?? '') && is_file($records), 409, 'A preparação CLI mudou; conservar o original e preparar novamente fora do servidor web.');
+        try { $report = json_decode(CheckedFile::read($preview), true, 512, JSON_THROW_ON_ERROR); }
+        catch (\JsonException) { abort(409, 'Prévia CLI inválida.'); }
+        abort_unless(is_array($report), 409, 'Prévia CLI inválida.');
+        abort_unless(($report['sha256'] ?? '') === $sha && ($report['record_count'] ?? null) === ($receipt['record_count'] ?? null)
+            && ($report['records_file_sha256'] ?? '') === ($receipt['records_sha256'] ?? '')
+            && (! $verifyRecords || CheckedFile::matches($records, $receipt['records_sha256'] ?? '')), 409, 'Resultados da preparação CLI alterados.');
+
+        return ['preview' => $preview, 'records' => $records, 'receipt' => $receipt];
+    }
+
+    /** @param array<string,array<string,mixed>> $files */
+    public function preparedSourcesReady(array $files): bool
+    {
+        try {
+            foreach ($files as $name => $expected) {
+                $source = $this->preparedSource($expected['sha256'], $name === 'Dados demograficos.xlsx' ? 2017 : null, false);
+                if ($source === null || $source['receipt']['record_count'] !== $expected['records']) { return false; }
+            }
+            return $files !== [];
+        } catch (\Throwable) { return false; }
+    }
+
     public function authorize(User $actor): void
     {
         $access = app(PortalProfileAccess::class);
@@ -43,13 +85,21 @@ final class DemographicImport
         $scan = app(MalwareScanner::class)->scan($path, (int) filesize($path), $disk->path($directory));
         abort_unless($scan['result'] === 'clean', 423, 'Análise antimalware pendente ou não limpa; original privado conservado.');
         $output = $disk->path($directory.'/preview.json');
-        $command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')];
-        if ($baselineYear !== null) {
-            array_push($command, '--baseline-year', (string) $baselineYear);
+        $prepared = $this->preparedSource($sha, $baselineYear);
+        if ($prepared !== null) {
+            abort_unless(copy($prepared['preview'], $output) && CheckedFile::matches($output, $prepared['receipt']['preview_sha256'])
+                && copy($prepared['records'], $disk->path($directory.'/records.ndjson'))
+                && CheckedFile::matches($disk->path($directory.'/records.ndjson'), $prepared['receipt']['records_sha256']), 409, 'Preparação CLI alterada durante a cópia.');
+        } else {
+            abort_unless(function_exists('proc_open'), 503, 'Este servidor web não executa o preparador. Prepare este original pelo importador CLI e confira o respectivo recibo antes de continuar.');
+            $command = [config('demography.python', 'python'), base_path('tools/demography_workbook.py'), $path, '--output', $output, '--records-output', $disk->path($directory.'/records.ndjson')];
+            if ($baselineYear !== null) {
+                array_push($command, '--baseline-year', (string) $baselineYear);
+            }
+            $process = new Process($command, base_path(), timeout: 120);
+            $process->run();
+            abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.');
         }
-        $process = new Process($command, base_path(), timeout: 120);
-        $process->run();
-        abort_unless($process->isSuccessful(), 422, 'Não foi possível reconhecer o Excel; confira a estrutura e o ano de referência.');
         $report = json_decode(CheckedFile::read($output), true, 512, JSON_THROW_ON_ERROR);
         abort_unless(hash_equals($sha, $report['sha256']) && CheckedFile::matches($path, $sha), 409);
         $summary = array_diff_key($report, ['records' => true]);

app/Services/DemographicInstallation.php

@@ -136,10 +136,14 @@ final class DemographicInstallation
         $tables = array_map(fn ($t) => Schema::hasTable($t), ['demographic_imports', 'demographic_observations', 'demographic_events']);
         $ledger = DB::table('migrations')->where('migration', self::MIGRATION)->exists();
         $consistent = ($ledger && ! in_array(false, $tables, true)) || (! $ledger && ! in_array(true, $tables, true));
-        $runtime = new Process([config('demography.python', 'python'), '-c', 'import sys; assert sys.version_info >= (3, 9)'], base_path(), timeout: 15);
+        $preparedFiles = self::FILES + self::remainingManifest()['files'] + self::zambeziaManifest()['files'];
+        $runtimeReady = app(DemographicImport::class)->preparedSourcesReady($preparedFiles);
         try {
-            $runtime->run();
-            $runtimeReady = $runtime->isSuccessful();
+            if (! $runtimeReady && function_exists('proc_open')) {
+                $runtime = new Process([config('demography.python', 'python'), '-c', 'import sys; assert sys.version_info >= (3, 9)'], base_path(), timeout: 15);
+                $runtime->run();
+                $runtimeReady = $runtime->isSuccessful();
+            }
         } catch (ExceptionInterface) {
             $runtimeReady = false;
         }

config/demography.php

@@ -5,6 +5,7 @@ return [
     'enabled' => env('DEMOGRAPHY_ENABLED', env('APP_URL') !== 'https://teste-investidor.tempo.co.mz'
         && in_array(env('APP_ENV', 'production'), ['local', 'testing'], true)),
     'source_directory' => env('DEMOGRAPHY_SOURCE_DIRECTORY', 'C:/Mac/Home/Documents/TEMPO/Portal Tempo/Servicos Tempo/portal do investidor/demografia/Provincias'),
+    'prepared_directory' => env('DEMOGRAPHY_PREPARED_DIRECTORY'),
     // An explicit interpreter avoids Windows Store execution aliases in web workers.
     'python' => env('DEMOGRAPHY_PYTHON', 'python'),
 ];

tests/Feature/DemographicInstallationTest.php

@@ -64,6 +64,15 @@ final class DemographicInstallationTest extends TestCase
         $this->assertStringContainsString('interpretador', $plan['notice']);
     }
 
+    public function test_disabled_process_support_does_not_crash_the_installation_page(): void
+    {
+        if (function_exists('proc_open')) { $this->markTestSkipped('Run this regression with -d disable_functions=proc_open.'); }
+        config(['demography.prepared_directory' => null]);
+        $this->actingAs($this->manager)->withSession(['portal.mfa_actor' => $this->manager->id, 'portal.mfa_at' => time()]);
+        $this->get('/management/demography/install')->assertOk()->assertInertia(fn (\Inertia\Testing\AssertableInertia $page) => $page
+            ->component('management/demography-install')->where('plan.runtime_ready', false)->where('plan.ready', false));
+    }
+
     public function test_schema_flag_alone_does_not_authorise_the_operational_target(): void
     {
         $this->app['env'] = 'local';

tests/Feature/DemographyTest.php

@@ -198,6 +198,41 @@ final class DemographyTest extends TestCase
         $migration->down();
     }
 
+    public function test_cli_preparation_imports_without_process_execution_and_keeps_exact_values(): void
+    {
+        Storage::fake('local');
+        $disk = Storage::disk('local');
+        $original = 'SYNTHETIC prepared original'; $sha = hash('sha256', $original);
+        $directory = 'prepared/'.$sha; $disk->makeDirectory($directory);
+        $disk->put('synthetic.xlsx', $original);
+        $row = ['province' => 'SYNTHETIC province', 'territory' => 'SYNTHETIC province', 'level' => 'province',
+            'year' => 2026, 'indicator' => 'population', 'sex' => 'all', 'residence' => 'all', 'age_group' => 'all', 'unit' => 'pessoas',
+            'data_kind' => 'projected', 'original_value' => '100.125', 'numeric_value' => '100.125', 'sheet' => '2026', 'cell' => 'B4'];
+        $records = json_encode($row, JSON_THROW_ON_ERROR)."\n";
+        $report = ['sha256' => $sha, 'source' => 'SYNTHETIC QA ONLY', 'record_count' => 1, 'records_file_sha256' => hash('sha256', $records),
+            'territories' => [['SYNTHETIC province', 'SYNTHETIC province', 'province']], 'issues' => [], 'issue_counts' => []];
+        $preview = json_encode($report, JSON_THROW_ON_ERROR);
+        $receipt = ['sha256' => $sha, 'baseline_year' => null, 'parser_sha256' => hash_file('sha256', base_path('tools/demography_workbook.py')),
+            'preview_sha256' => hash('sha256', $preview), 'records_sha256' => hash('sha256', $records), 'record_count' => 1];
+        $disk->put($directory.'/preview.json', $preview); $disk->put($directory.'/records.ndjson', $records);
+        $disk->put($directory.'/preparation.json', json_encode($receipt, JSON_THROW_ON_ERROR));
+        config(['demography.prepared_directory' => $disk->path('prepared'), 'demography.python' => '/missing-python-must-not-run']);
+        $this->mock(\App\Services\Malware\MalwareScanner::class)->shouldReceive('scan')->once()->andReturn([
+            'result' => 'clean', 'engine' => 'synthetic', 'engine_version' => 'test']);
+        $service = app(DemographicImport::class);
+        $this->assertTrue($service->preparedSourcesReady(['synthetic.xlsx' => ['sha256' => $sha, 'records' => 1]]));
+        $result = $service->prepare($this->manager, $disk->path('synthetic.xlsx'), 'synthetic.xlsx', 'SYNTHETIC', null);
+        $this->assertSame(0, DB::table('demographic_observations')->where('import_id', $result['id'])->count());
+        $service->confirm($this->manager, $result['id'], $sha, [], 'Synthetic authenticated import from CLI preparation.');
+        $this->assertSame('100.125', DB::table('demographic_observations')->where('import_id', $result['id'])->value('numeric_value'));
+        $this->assertSame('imported', DB::table('demographic_imports')->where('id', $result['id'])->value('status'));
+        $disk->put($directory.'/records.ndjson', 'TAMPERED');
+        try { $service->preparedSource($sha, null); $this->fail('Changed prepared records must be refused.'); }
+        catch (\Symfony\Component\HttpKernel\Exception\HttpException $error) { $this->assertSame(409, $error->getStatusCode()); }
+        $receipt['parser_sha256'] = str_repeat('0', 64); $disk->put($directory.'/preparation.json', json_encode($receipt, JSON_THROW_ON_ERROR));
+        $this->assertFalse($service->preparedSourcesReady(['synthetic.xlsx' => ['sha256' => $sha, 'records' => 1]]));
+    }
+
     public function test_remote_consultation_requires_explicit_activation_and_preserves_versions(): void
     {
         $environment = app()->environment();

tools/demography_workbook.py

@@ -267,7 +267,10 @@ if __name__ == "__main__":
     cli.add_argument("--baseline-year", type=int)
     cli.add_argument("--output", required=True)
     cli.add_argument("--records-output")
+    cli.add_argument("--preparation-receipt")
     args = cli.parse_args()
+    if args.preparation_receipt and not args.records_output:
+        cli.error('--preparation-receipt requires --records-output')
     result = parse(args.file, args.baseline_year)
     if args.records_output:
         record_path = Path(args.records_output)
@@ -276,4 +279,10 @@ if __name__ == "__main__":
                 target.write(json.dumps(record, ensure_ascii=False, separators=(',', ':'))+'\n')
         result['records_file_sha256'] = hashlib.sha256(record_path.read_bytes()).hexdigest()
     Path(args.output).write_text(json.dumps(result, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+    if args.preparation_receipt:
+        receipt = dict(sha256=result['sha256'], baseline_year=args.baseline_year,
+                       parser_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+                       preview_sha256=hashlib.sha256(Path(args.output).read_bytes()).hexdigest(),
+                       records_sha256=result['records_file_sha256'], record_count=result['record_count'])
+        Path(args.preparation_receipt).write_text(json.dumps(receipt, separators=(',', ':')), encoding='utf-8')
     print(json.dumps({k: v for k, v in result.items() if k not in ("records", "issues")}, ensure_ascii=False))