From 439918d143d24334b26cdb0387addafdcf835f73 Mon Sep 17 00:00:00 2001 From: nala Date: Mon, 28 Sep 2026 20:48:34 +0200 Subject: [PATCH] feat: audit cartography workbook with provenance and quarantine --- app/Services/CartographyWorkbookAudit.php | 142 ++++++++++++++++++ docs/investmoz/gis/README.md | 92 ++++++++++++ .../gis/source-manifest.example.json | 11 ++ tests/Unit/CartographyWorkbookAuditTest.php | 75 +++++++++ tools/cartography-audit.php | 20 +++ 5 files changed, 340 insertions(+) create mode 100644 app/Services/CartographyWorkbookAudit.php create mode 100644 docs/investmoz/gis/README.md create mode 100644 docs/investmoz/gis/source-manifest.example.json create mode 100644 tests/Unit/CartographyWorkbookAuditTest.php create mode 100644 tools/cartography-audit.php diff --git a/app/Services/CartographyWorkbookAudit.php b/app/Services/CartographyWorkbookAudit.php new file mode 100644 index 00000000..d77a58c1 --- /dev/null +++ b/app/Services/CartographyWorkbookAudit.php @@ -0,0 +1,142 @@ + $manifest + * @return array + */ + public function inspect(string $path, array $manifest): array + { + if (! is_file($path) || filesize($path) > 20_000_000) { + throw new RuntimeException('Workbook missing or larger than 20 MB.'); + } + $checksum = hash_file('sha256', $path); + if ($checksum === false) { + throw new RuntimeException('Unable to read workbook checksum.'); + } + if (! isset($manifest['sha256']) || ! hash_equals(strtolower((string) $manifest['sha256']), $checksum)) { + throw new RuntimeException('Manifest checksum does not match the workbook.'); + } + foreach (['inspection_authorization'] as $key) { + if (! is_string($manifest[$key] ?? null) || trim($manifest[$key]) === '') { + throw new RuntimeException('Missing manifest field: '.$key); + } + } + if (isset($manifest['classification']) && ! in_array($manifest['classification'], ['public', 'internal', 'confidential', 'restricted'], true)) { + throw new RuntimeException('Invalid classification.'); + } + $zip = new ZipArchive; + if ($zip->open($path) !== true) { + throw new RuntimeException('Invalid XLSX container.'); + } + try { + $bytes = 0; + if ($zip->numFiles > 1000) { + throw new RuntimeException('Too many archive entries.'); + } + for ($i = 0; $i < $zip->numFiles; $i++) { + $entry = $zip->statIndex($i); + if ($entry === false) { + throw new RuntimeException('Unreadable archive entry.'); + } + $bytes += $entry['size']; + if ($bytes > 50_000_000 || str_contains($entry['name'], '../')) { + throw new RuntimeException('Unsafe or oversized archive.'); + } + } + $strings = []; + if ($zip->locateName('xl/sharedStrings.xml') !== false) { + foreach ($this->xml($zip, 'xl/sharedStrings.xml')->xpath('//*[local-name()="si"]') ?: [] as $item) { + $strings[] = implode('', array_map(fn ($t) => (string) $t, $item->xpath('.//*[local-name()="t"]') ?: [])); + } + } + $relationships = []; + foreach ($this->xml($zip, 'xl/_rels/workbook.xml.rels')->children() as $rel) { + $target = (string) $rel['Target']; + if ((string) $rel['TargetMode'] !== 'External' && ! str_contains($target, '..')) { + $relationships[(string) $rel['Id']] = str_starts_with($target, '/') ? ltrim($target, '/') : 'xl/'.$target; + } + } + $sheets = []; + $rowCount = 0; + foreach ($this->xml($zip, 'xl/workbook.xml')->xpath('//*[local-name()="sheet"]') ?: [] as $sheet) { + $id = (string) $sheet->attributes('http://schemas.openxmlformats.org/officeDocument/2006/relationships')['id']; + if (! isset($relationships[$id])) { + throw new RuntimeException('Unresolved worksheet relationship.'); + } + $rows = []; + $seen = []; + foreach ($this->xml($zip, $relationships[$id])->xpath('//*[local-name()="sheetData"]/*[local-name()="row"]') ?: [] as $row) { + if (++$rowCount > 50000) { + throw new RuntimeException('Audit limit of 50000 rows exceeded.'); + } + $values = []; + $issues = []; + foreach ($row->children() as $cell) { + $address = (string) $cell['r']; + $value = (string) $cell->v; + if ((string) $cell['t'] === 's') { + $value = $strings[(int) $value] ?? ''; + } elseif ((string) $cell['t'] === 'inlineStr') { + $value = implode('', array_map(fn ($t) => (string) $t, $cell->xpath('.//*[local-name()="t"]') ?: [])); + } + $values[preg_replace('/[0-9]+$/', '', $address)] = $value; + if ($value === '') { + $issues[] = ['field' => $address, 'code' => 'empty_cell']; + } + if (isset($cell->f)) { + $issues[] = ['field' => $address, 'code' => 'formula_not_evaluated']; + } + if ((string) $cell['t'] === 'e') { + $issues[] = ['field' => $address, 'code' => 'spreadsheet_error']; + } + } + $hash = hash('sha256', json_encode($values, JSON_THROW_ON_ERROR)); + $duplicate = isset($seen[$hash]); + $seen[$hash] = true; + $issues[] = ['field' => '*', 'code' => 'semantic_mapping_and_institutional_validation_pending']; + if (empty($manifest['original_crs'])) { + $issues[] = ['field' => '*', 'code' => 'original_crs_unknown']; + } + $rows[] = ['row' => (int) $row['r'], 'status' => $duplicate ? 'duplicate' : 'pending', 'populated_columns' => array_keys($values), 'issues' => $issues]; + } + $sheets[] = ['name' => (string) $sheet['name'], 'state' => (string) $sheet['state'] ?: 'visible', 'rows' => $rows]; + } + + return ['sha256' => $checksum, 'version' => $manifest['version'] ?? null, 'classification' => $manifest['classification'] ?? null, + 'pending_metadata' => array_values(array_filter(['owner', 'version', 'received_at', 'classification', 'original_crs'], fn ($key) => empty($manifest[$key]))), + 'original_crs' => $manifest['original_crs'] ?? null, 'publication_allowed' => false, 'transformations' => [], + 'status' => 'quarantined', 'sheets' => $sheets, 'rows_inspected' => $rowCount, + 'limitations' => ['No semantic header mapping, omitted-cell inference, coordinate validation or external reference resolution.', 'No source values, coordinates or formulas copied to the report.']]; + } finally { + $zip->close(); + } + } + + private function xml(ZipArchive $zip, string $name): SimpleXMLElement + { + $content = $zip->getFromName($name); + if ($content === false || preg_match('/path = tempnam(sys_get_temp_dir(), 'gis-test-'); + $zip = new ZipArchive; + $zip->open($this->path, ZipArchive::OVERWRITE); + $zip->addFromString('xl/workbook.xml', ''); + $zip->addFromString('xl/_rels/workbook.xml.rels', ''); + $zip->addFromString('xl/worksheets/sheet1.xml', 'Private synthetic labelPrivate synthetic label1+12'); + $zip->close(); + } + + protected function tearDown(): void + { + unlink($this->path); + parent::tearDown(); + } + + /** @return array */ + private function manifest(): array + { + return ['sha256' => hash_file('sha256', $this->path), 'owner' => 'Synthetic owner', 'version' => 'test-1', + 'received_at' => '2026-09-28', 'classification' => 'internal', 'inspection_authorization' => 'Synthetic test only']; + } + + public function test_audit_quarantines_unknown_crs_and_reports_duplicates_without_copying_source_values(): void + { + $report = (new CartographyWorkbookAudit)->inspect($this->path, $this->manifest()); + $this->assertSame(3, $report['rows_inspected']); + $this->assertSame('quarantined', $report['status']); + $this->assertFalse($report['publication_allowed']); + $this->assertSame([], $report['transformations']); + $this->assertSame('duplicate', $report['sheets'][0]['rows'][1]['status']); + $this->assertStringNotContainsString('Private synthetic label', json_encode($report)); + $this->assertStringContainsString('formula_not_evaluated', json_encode($report)); + $this->assertStringContainsString('original_crs_unknown', json_encode($report)); + } + + public function test_checksum_mismatch_is_rejected(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('checksum'); + (new CartographyWorkbookAudit)->inspect($this->path, [...$this->manifest(), 'sha256' => str_repeat('0', 64)]); + } + + public function test_inspection_requires_authorization_evidence(): void + { + $this->expectException(RuntimeException::class); + (new CartographyWorkbookAudit)->inspect($this->path, [...$this->manifest(), 'inspection_authorization' => '']); + } + + public function test_xml_entities_are_rejected(): void + { + $zip = new ZipArchive; + $zip->open($this->path); + $zip->addFromString('xl/workbook.xml', ']>&x;'); + $zip->close(); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('unsafe'); + (new CartographyWorkbookAudit)->inspect($this->path, $this->manifest()); + } +} diff --git a/tools/cartography-audit.php b/tools/cartography-audit.php new file mode 100644 index 00000000..1a61159e --- /dev/null +++ b/tools/cartography-audit.php @@ -0,0 +1,20 @@ + 100000) { + throw new RuntimeException('Set INVESTMOZ_CARTOGRAPHY_FILE and INVESTMOZ_CARTOGRAPHY_MANIFEST to authorised local files.'); + } + $manifest = json_decode(file_get_contents($manifestPath), true, 32, JSON_THROW_ON_ERROR); + $report = (new CartographyWorkbookAudit)->inspect($workbookPath, $manifest); + echo json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR).PHP_EOL; +} catch (Throwable $exception) { + fwrite(STDERR, 'Audit failed: '.$exception->getMessage().PHP_EOL); + exit(1); +}