'testing', 'APP_KEY' => 'base64:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', 'DB_CONNECTION' => 'pgsql', 'DB_HOST' => '127.0.0.1', 'DB_PORT' => '55441', 'DB_DATABASE' => $target['database'], 'DB_USERNAME' => 'postgres', 'DB_PASSWORD' => '', 'DB_URL' => '', 'CACHE_STORE' => 'array', 'SESSION_DRIVER' => 'array'] as $key => $value) { putenv($key.'='.$value); $_ENV[$key] = $_SERVER[$key] = $value; } require $root.'/vendor/autoload.php'; $app = require $root.'/bootstrap/app.php'; $app->make(Kernel::class)->bootstrap(); $identity = DB::selectOne('SELECT current_database() AS database,inet_server_port() AS port'); if (! app()->environment('testing') || $identity->database !== $target['database'] || (int) $identity->port !== 55441) { throw new RuntimeException('Non-QA target rejected before writes.'); } $action = $argv[1] ?? 'preview'; $level = (int) ($argv[2] ?? 1); if (! in_array($level, [1, 2, 3], true)) { throw new RuntimeException('Administrative levels 1-3 only.'); } if (! in_array($action, ['preview', 'import'], true)) { throw new RuntimeException('Only preview/import are supported.'); } $file = $directory.'/moz-adm'.$level.'-ine-20190607.geojson'; $metadataFile = $directory.'/moz-adm'.$level.'-ine-metadata.html'; $sourceHash = hash_file('sha256', $file); $sourceScan = null; if ($action === 'import') { $proofs = json_decode(file_get_contents($directory.'/geometry-source-scans.json'), true, 512, JSON_THROW_ON_ERROR); foreach ($proofs['files'] as $scan) { if ($scan['file'] === basename($file) && $scan['sha256'] === $sourceHash) { $sourceScan = ($scan['scan']['result'] ?? null) === 'clean' ? $scan['scan'] : ($scan['gis_scan'] ?? null); } } if (($sourceScan['result'] ?? null) !== 'clean' || strtotime($proofs['captured_at']) < time() - 86400) { throw new RuntimeException('Current clean scan tied to the retained original hash required before QA import.'); } } $metadata = file_get_contents($metadataFile); if (! str_contains($metadata, 'Mozambique INE - Instituto Nacional de Estatistica') || ! str_contains($metadata, '2019-06-07')) { throw new RuntimeException('Source identity/version absent from retained metadata.'); } $source = json_decode(file_get_contents($file), true, 512, JSON_THROW_ON_ERROR); if (($source['type'] ?? '') !== 'FeatureCollection' || count($source['features'] ?? []) !== [1 => 11, 2 => 159, 3 => 411][$level] || ($source['exceededTransferLimit'] ?? false)) { throw new RuntimeException('Incomplete/unexpected provincial dataset.'); } $normalize = fn (string $value) => strtolower(trim(Str::ascii($value))); $locations = DB::table('geo_locations')->where('location_type', [1 => 'province', 2 => 'district', 3 => 'admin_post'][$level])->whereNull('deleted_at')->get(['id', 'name', 'parent_id', 'code', 'source_code']); $results = []; foreach ($source['features'] as $feature) { $properties = $feature['properties']; $name = $properties['ADM'.$level.'_PT']; $parent = $level > 1 ? DB::table('geo_geometries')->where('source_system', 'ine-cod-ab')->where('external_id', '2019-06-07:adm'.($level - 1).':'.$properties['ADM'.($level - 1).'_PCODE'])->whereNull('deleted_at')->value('geo_location_id') : null; $matches = $locations->filter(fn ($location) => $normalize($location->name) === $normalize($name) && ($level === 1 || ($parent !== null && $location->parent_id === $parent)))->values(); $results[] = ['pcode' => $properties['ADM'.$level.'_PCODE'], 'source_name' => $name, 'source_parent_code' => $level > 1 ? $properties['ADM'.($level - 1).'_PCODE'] : null, 'candidates' => $matches->all(), 'matched_id' => count($matches) === 1 ? $matches[0]->id : null, 'match_basis' => count($matches) === 1 ? 'unique_exact_name_and_matched_parent_after_case_and_diacritics' : 'unresolved_no_implicit_translation_or_parent_mapping', 'geometry_type' => $feature['geometry']['type'], 'feature' => $feature]; } $inserted = 0; $reused = 0; if ($action === 'import') { DB::transaction(function () use (&$results, &$inserted, &$reused, $sourceHash, $metadataFile, $level, $sourceScan) { DB::select('SELECT pg_advisory_xact_lock(91026, 20190607)'); foreach ($results as &$row) { if ($row['matched_id'] === null) { continue; } $geometry = json_encode($row['feature']['geometry'], JSON_THROW_ON_ERROR); $type = strtolower($row['geometry_type']); $valid = DB::selectOne('SELECT ST_IsValid(g) AS valid, ST_IsEmpty(g) AS empty, GeometryType(g) AS type FROM (SELECT ST_SetSRID(ST_GeomFromGeoJSON(?),4326) AS g) x', [$geometry]); if (! $valid->valid || $valid->empty || ! in_array($type, ['polygon', 'multipolygon'], true)) { $row['matched_id'] = null; $row['match_basis'] = 'invalid_geometry_no_automatic_repair'; continue; } $external = '2019-06-07:adm'.$level.':'.$row['pcode']; $existing = DB::table('geo_geometries')->where('source_system', 'ine-cod-ab')->where('external_id', $external)->first(); if ($existing) { $saved = json_decode($existing->properties, true, 512, JSON_THROW_ON_ERROR); if ($existing->geo_location_id !== $row['matched_id'] || ($saved['source_sha256'] ?? null) !== $sourceHash || $existing->deleted_at !== null) { throw new RuntimeException('Identity/version conflict; nothing overwritten.'); } $reused++; continue; } $trace = ['authority' => 'Mozambique INE - Instituto Nacional de Estatistica', 'provider' => 'UNOCHA; exemplar hosted by World Bank', 'source_version' => '2019-06-07', 'source_url' => 'https://services.arcgis.com/iQ1dY19aHwbSDYIF/arcgis/rest/services/Mozambique/FeatureServer/'.($level + 2), 'source_sha256' => $sourceHash, 'metadata_sha256' => hash_file('sha256', $metadataFile), 'source_properties' => $row['feature']['properties'], 'match_basis' => $row['match_basis'], 'qa_only' => true, 'purpose' => 'administrative_boundary', 'legal_currentness' => 'not_determined', 'investment_site' => false, 'source_scan' => $sourceScan]; DB::insert('INSERT INTO geo_geometries(id,source_system,external_id,geo_location_id,geometry_type,geom,properties,data_classification) VALUES (?,?,?,?,?,ST_SetSRID(ST_GeomFromGeoJSON(?),4326),?::jsonb,?)', [(string) Str::uuid(), 'ine-cod-ab', $external, $row['matched_id'], $type, $geometry, json_encode($trace, JSON_THROW_ON_ERROR), 'internal']); $inserted++; } if ($inserted > 0) { DB::table('audit_events')->insert(['source_system' => 'portal-audit-qa', 'entity_type' => 'geo_geometries', 'action' => 'official_geometry_qa_import', 'data_classification' => 'internal', 'after_data' => json_encode(['level' => $level, 'source_sha256' => $sourceHash, 'inserted' => $inserted, 'published' => false], JSON_THROW_ON_ERROR)]); } }); } foreach ($results as &$row) { unset($row['feature']); } $saved = DB::select("SELECT external_id,geo_location_id,data_classification,ST_IsValid(geom) AS valid,ST_SRID(geom) AS srid,ST_NPoints(geom) AS vertices,properties->>'source_sha256' AS source_sha256 FROM geo_geometries WHERE source_system='ine-cod-ab' ORDER BY external_id"); $result = ['captured_at' => gmdate('c'), 'action' => $action, 'level' => $level, 'target' => $target, 'source_sha256' => $sourceHash, 'metadata_sha256' => hash_file('sha256', $metadataFile), 'source_features' => count($source['features']), 'mappings' => $results, 'inserted' => $inserted, 'reused' => $reused, 'recovered_geometries' => $saved, 'active_writes' => 0, 'published' => false, 'visual_verified' => false, 'remaining' => ['Unmatched names/parents listed per feature; no fuzzy mapping', 'No locality polygons in the identified official exemplar', 'Private QA import does not populate public map layers or investment locations']]; $destination = $directory.'/geometry-qa-adm'.$level.'-'.$action.'.json'; if (is_file($destination)) { copy($destination, $directory.'/geometry-qa-adm'.$level.'-'.$action.'-'.gmdate('Ymd_His').'-'.bin2hex(random_bytes(3)).'.json'); } file_put_contents($destination, json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR)); echo json_encode(['action' => $action, 'level' => $level, 'features' => count($results), 'matched' => count(array_filter($results, fn ($row) => $row['matched_id'] !== null)), 'inserted' => $inserted, 'reused' => $reused, 'persisted_total' => count($saved), 'target' => $target, 'active_writes' => 0], JSON_THROW_ON_ERROR).PHP_EOL;