provincias
tests / ci (push) Canceled after 0s

This commit is contained in:
2026-09-29 14:14:06 +02:00
parent e780e67e98
commit 67dad461d9
35 changed files with 1046 additions and 42 deletions
+14 -5
View File
@@ -15,9 +15,11 @@ $expected = '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff';
if ($data['checksum'] !== $expected || hash_file('sha256', 'C:/Mac/Home/Documents/eBAU/cartografia_moz.xlsx') !== $expected) {
throw new RuntimeException('Authorised source checksum mismatch.');
}
$created = DB::transaction(function () use ($data, $expected) {
$counts = DB::transaction(function () use ($data, $expected) {
DB::select('SELECT pg_advisory_xact_lock(280926, 150000)');
$created = 0;
$updated = 0;
$unchanged = 0;
foreach ($data['nodes'] as $node) {
$existing = DB::table('geo_locations')->where('id', $node['id'])->first();
if ($existing) {
@@ -27,6 +29,13 @@ $created = DB::transaction(function () use ($data, $expected) {
}
}
if ($existing->source_code !== $node['source_code']) {
DB::table('geo_locations')->where('id', $node['id'])->update(['source_code' => $node['source_code'], 'updated_at' => now()]);
$updated++;
} else {
$unchanged++;
}
continue;
}
if ($node['parent_id'] && ! DB::table('geo_locations')->where('id', $node['parent_id'])->exists()) {
@@ -35,12 +44,12 @@ $created = DB::transaction(function () use ($data, $expected) {
DB::table('geo_locations')->insert([...$node, 'source_system' => 'cartografia-workbook', 'external_id' => $expected.':'.$node['id'], 'data_classification' => 'internal']);
$created++;
}
if ($created) {
if ($created || $updated) {
DB::table('audit_events')->insert(['source_system' => 'cartografia-workbook', 'entity_type' => 'geo_locations', 'action' => 'owner_authorised_reference_import',
'after_data' => json_encode(['checksum' => $expected, 'created' => $created, 'rejected_rows' => $data['rejected_rows'],
'after_data' => json_encode(['checksum' => $expected, 'created' => $created, 'updated' => $updated, 'rejected_rows' => $data['rejected_rows'],
'approval_basis' => 'workspace_user_instructions', 'codes' => 'internal_identifiers_not_official_ine', 'classification' => 'internal'], JSON_THROW_ON_ERROR)]);
}
return $created;
return compact('created', 'updated', 'unchanged');
});
echo json_encode(['created' => $created, 'rejected_rows' => count($data['rejected_rows']), 'scope' => 'internal_management_dropdowns'], JSON_THROW_ON_ERROR).PHP_EOL;
echo json_encode([...$counts, 'read' => $data['read'], 'ignored' => $data['ignored_sub_locality_rows'] + $data['duplicates'], 'rejected_rows' => count($data['rejected_rows']), 'errors_file' => $path, 'scope' => 'internal_management_dropdowns'], JSON_THROW_ON_ERROR).PHP_EOL;
+43
View File
@@ -0,0 +1,43 @@
"""Extract the user-supplied SQL workbook without executing it."""
import hashlib
import json
from pathlib import Path
import xml.etree.ElementTree as ET
import zipfile
source = Path('C:/Mac/Home/Documents/TEMPO/Portal Tempo/Servicos Tempo/prompt/cartografia_moz.xlsx')
checksum = hashlib.sha256(source.read_bytes()).hexdigest()
if checksum != '1e36c769c4767420b10f1ff5ee0c2955cdb1b03c98179c5cad549e832ffb1f15':
raise ValueError('Source changed; inspect and approve the new mapping before execution')
ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
with zipfile.ZipFile(source) as archive:
strings = [''.join(item.itertext()) for item in ET.fromstring(archive.read('xl/sharedStrings.xml')).findall('s:si', ns)]
def cells(number):
result = {}
for cell in ET.fromstring(archive.read(f'xl/worksheets/sheet{number}.xml')).findall('.//s:c', ns):
value = cell.findtext('s:v', '', ns)
if cell.attrib.get('t') == 's':
value = strings[int(value)]
elif cell.attrib.get('t') == 'inlineStr':
value = ''.join(cell.find('s:is', ns).itertext())
if cell.find('s:f', ns) is not None:
raise ValueError('SQL package must not depend on evaluated formulas')
result[cell.attrib['r']] = value
return result
ddl_cells, insert_cells = cells(3), cells(10)
ddl = [ddl_cells[f'A{row}'] for row in range(1, 15)]
statements = [insert_cells[f'B{row}'] for row in range(2, 21416)]
tables = ['provincia', 'distrito', 'posto_administrativo', 'localidade', 'bairro_urbano', 'aldeia_quarteirao']
counts = {table: sum(sql.startswith('INSERT INTO ' + table + ' ') for sql in statements) for table in tables}
expected = dict(zip(tables, [11, 150, 451, 1377, 2838, 16583]))
if counts != expected or statements[2] != 'BEGIN;' or statements[-1] != 'COMMIT;':
raise ValueError('Unexpected transaction or counts')
quality = cells(11)
report = {'source_sha256': checksum, 'ddl_cells': 'SQL_DDL!A1:A14', 'insert_cells': 'SQL_INSERTS!B2:B21415',
'counts': counts, 'total': sum(counts.values()), 'auxiliary_parents_from_source': 1075,
'quality_rows': 1147, 'has_coordinates': False,
'warning': '1075 parents explicitly marked NAO INFORMADO by the workbook; these are unresolved references, not authenticated administrative designations.'}
output = Path('storage/app/private/cartography-sql-package.json')
output.write_text(json.dumps({'report': report, 'ddl': ddl, 'statements': statements, 'quality_cells': quality}, ensure_ascii=False), encoding='utf-8')
Path('docs/investmoz/gis/sql-workbook-inspection.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(json.dumps(report))
+18 -5
View File
@@ -5,6 +5,7 @@ from pathlib import Path
import uuid
import xml.etree.ElementTree as ET
import zipfile
import unicodedata
source = Path('C:/Mac/Home/Documents/eBAU/cartografia_moz.xlsx')
checksum = hashlib.sha256(source.read_bytes()).hexdigest()
@@ -12,15 +13,24 @@ if checksum != '964a187ad32ea553b9e112cd159caa90b900050c42b7329594a75bcd798a96ff
raise ValueError('Authorised workbook changed')
ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
country = str(uuid.uuid5(uuid.NAMESPACE_URL, 'investmoz:workbook:MZ'))
nodes = {(): dict(id=country, parent_id=None, name='Moçambique', location_type='country', code='508')}
nodes = {(): dict(id=country, parent_id=None, name='Moçambique', location_type='country', code='508', source_code='508')}
definitions = {}
conflicts = set()
rejected = []
ignored = 0
read = 0
duplicates = 0
with zipfile.ZipFile(source) as archive:
strings = [''.join(si.itertext()) for si in ET.fromstring(archive.read('xl/sharedStrings.xml')).findall('s:si', ns)]
sheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
headers = []
for cell in sheet.findall('s:sheetData/s:row', ns)[0]:
value = cell.findtext('s:v', '', ns)
headers.append(strings[int(value)] if cell.attrib.get('t') == 's' else value)
if len(headers) < 8 or 'designa' not in unicodedata.normalize('NFKD', headers[7]).casefold():
raise ValueError('Unexpected workbook headers; inspect before importing')
for row in sheet.findall('s:sheetData/s:row', ns)[1:]:
read += 1
cells = {}
invalid = False
for cell in row:
@@ -32,7 +42,7 @@ with zipfile.ZipFile(source) as archive:
value = ''.join(cell.find('s:is', ns).itertext())
if cell.find('s:f', ns) is not None or cell.attrib.get('t') == 'e':
invalid = True
cells[column] = ' '.join(value.split())
cells[column] = unicodedata.normalize('NFC', ' '.join(value.split()))
if cells.get('F') or cells.get('G'):
ignored += 1
continue
@@ -47,11 +57,14 @@ with zipfile.ZipFile(source) as archive:
if key in conflicts:
rejected.append({'row': row.attrib['r'], 'reason': 'conflicting_designation'})
continue
if key in definitions and definitions[key]['name'] != name:
if key in definitions and definitions[key]['name'].casefold() != name.casefold():
rejected.extend([{'row': definitions[key]['row'], 'reason': 'conflicting_designation'}, {'row': row.attrib['r'], 'reason': 'conflicting_designation'}])
conflicts.add(key)
del definitions[key]
continue
if key in definitions:
duplicates += 1
continue
definitions[key] = {'name': name, 'row': row.attrib['r']}
for key, definition in sorted(definitions.items(), key=lambda item: (len(item[0]), item[0])):
if key[:-1] not in nodes:
@@ -60,10 +73,10 @@ for key, definition in sorted(definitions.items(), key=lambda item: (len(item[0]
parent = nodes[key[:-1]]['id']
kind = ['province', 'district', 'admin_post', 'locality'][len(key)-1]
identifier = str(uuid.uuid5(uuid.NAMESPACE_URL, parent + ':' + kind + ':' + key[-1].casefold()))
nodes[key] = dict(id=identifier, parent_id=parent, name=definition['name'], location_type=kind, code='XLSX-' + identifier)
nodes[key] = dict(id=identifier, parent_id=parent, name=definition['name'], location_type=kind, code='XLSX-' + identifier, source_code=key[-1])
output = Path('storage/app/private/territory-workbook-candidates.json')
previous = output.with_name('territory-workbook-candidates-before-designation-fix.json')
if output.exists() and not previous.exists():
previous.write_bytes(output.read_bytes())
output.write_text(json.dumps(dict(checksum=checksum, mapping_version=2, nodes=list(nodes.values()), rejected_rows=rejected, ignored_sub_locality_rows=ignored), ensure_ascii=False), encoding='utf-8')
output.write_text(json.dumps(dict(checksum=checksum, mapping_version=3, headers=headers, read=read, duplicates=duplicates, nodes=list(nodes.values()), rejected_rows=rejected, ignored_sub_locality_rows=ignored), ensure_ascii=False), encoding='utf-8')
print(json.dumps(dict(counts={kind: sum(n['location_type'] == kind for n in nodes.values()) for kind in ['country','province','district','admin_post','locality']}, rejected_rows=len(rejected), ignored_sub_locality_rows=ignored)))
+24
View File
@@ -0,0 +1,24 @@
<?php
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\DB;
require __DIR__.'/../vendor/autoload.php';
$app = require __DIR__.'/../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
$db = DB::connection();
$report = $db->transaction(function () use ($db) {
$db->statement('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY');
return [
'options_query_plan' => $db->select("EXPLAIN (FORMAT JSON) SELECT id,name FROM geo_locations WHERE deleted_at IS NULL AND parent_id=? AND location_type='district' ORDER BY name LIMIT 100", [$db->table('geo_locations')->where('location_type', 'province')->whereNull('deleted_at')->value('id')]),
'counts' => $db->select('SELECT location_type,count(*) FROM geo_locations WHERE deleted_at IS NULL GROUP BY location_type ORDER BY location_type'),
'duplicate_names' => $db->select('SELECT parent_id,location_type,lower(trim(name)) as name,count(*) FROM geo_locations WHERE deleted_at IS NULL GROUP BY parent_id,location_type,lower(trim(name)) HAVING count(*)>1'),
'invalid_parents' => $db->select("SELECT g.id,g.location_type FROM geo_locations g LEFT JOIN geo_locations p ON p.id=g.parent_id WHERE g.deleted_at IS NULL AND (g.location_type='country' AND g.parent_id IS NOT NULL OR g.location_type IN ('province','district','admin_post','locality') AND (p.id IS NULL OR p.deleted_at IS NOT NULL OR p.location_type <> CASE g.location_type WHEN 'province' THEN 'country' WHEN 'district' THEN 'province' WHEN 'admin_post' THEN 'district' WHEN 'locality' THEN 'admin_post' END))"),
'sectors' => $db->table('sectors')->whereNull('deleted_at')->orderBy('id')->get(['id', 'name', 'slug']),
'php' => PHP_VERSION, 'laravel' => app()->version(),
'postgres' => $db->selectOne('SHOW server_version')->server_version,
'postgis' => $db->selectOne('SELECT postgis_lib_version() as version')->version,
];
});
echo json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR).PHP_EOL;
+110
View File
@@ -0,0 +1,110 @@
// Synthetic QA server only; never writes to the application database.
import fs from 'node:fs';
const target = await (await fetch('http://127.0.0.1:9224/json/new?about:blank', { method: 'PUT' })).json();
const ws = new WebSocket(target.webSocketDebuggerUrl);
await new Promise(resolve => ws.addEventListener('open', resolve, { once: true }));
let sequence = 0;
const pending = new Map();
const errors = [];
const requests = [];
const responses = [];
const responseIds = new Set();
const responseBodies = [];
ws.onmessage = event => {
const message = JSON.parse(event.data);
if (message.method === 'Network.requestWillBeSent' && message.params.request.url.includes('/management/opportunities') && message.params.request.postData) {
const request = message.params.request; let data = {};
try { data = JSON.parse(request.postData); } catch {}
const keys = ['province_id','district_id','admin_post_id','locality_id','territory_version','reference'];
requests.push({ method: request.method, path: new URL(request.url).pathname, payload: Object.fromEntries(keys.filter(k => k in data).map(k => [k, data[k]])) });
}
if (message.method === 'Network.responseReceived' && message.params.response.url.includes('/management/opportunities')) {
responses.push({ path: new URL(message.params.response.url).pathname, status: message.params.response.status });
responseIds.add(message.params.requestId);
}
if (message.method === 'Network.loadingFinished' && responseIds.has(message.params.requestId)) {
send('Network.getResponseBody', { requestId: message.params.requestId }).then(result => {
try { const body = JSON.parse(result.body); responseBodies.push({component: body.component, errors: body.props?.errors}); } catch {}
}).catch(() => {});
}
if (message.method === 'Runtime.exceptionThrown') errors.push(message.params.exceptionDetails.text);
if (message.id) { const callback = pending.get(message.id); pending.delete(message.id); message.error ? callback.reject(message.error) : callback.resolve(message.result); }
};
const send = (method, params = {}) => new Promise((resolve, reject) => { const id = ++sequence; pending.set(id, { resolve, reject }); ws.send(JSON.stringify({ id, method, params })); });
const evaluate = async expression => { const result = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }); if (result.exceptionDetails) throw Error(JSON.stringify(result.exceptionDetails)); return result.result?.value; };
const wait = async expression => { for (let n = 0; n < 100; n++) { if (await evaluate(expression)) return; await new Promise(resolve => setTimeout(resolve, 100)); } throw Error('Timed out: ' + expression); };
const select = async (index, value) => evaluate(`(()=>{const element=document.querySelectorAll('select')[${index}];Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype,'value').set.call(element,${JSON.stringify(value)});element.dispatchEvent(new Event('change',{bubbles:true}));})()`);
try {
await send('Runtime.enable'); await send('Page.enable'); await send('Network.enable');
await send('Page.navigate', { url: 'http://127.0.0.1:18027/login' });
await wait(`document.readyState==='complete' && (location.pathname!='/login' || !!document.querySelector('input[type="password"]'))`);
if (await evaluate(`location.pathname==='/login'`)) {
for (const [selector, text] of [['input[type="email"]', 'mne-browser@example.test'], ['input[type="password"]', 'SyntheticMneOnly-2026!']]) { await evaluate(`document.querySelector(${JSON.stringify(selector)}).focus()`); await send('Input.insertText', { text }); }
await evaluate(`document.querySelector('button[type="submit"]').click()`); await wait(`!location.pathname.startsWith('/login')`);
}
const token = 'geo-browser-' + Date.now();
const observations = [];
await send('Page.navigate', { url: 'http://127.0.0.1:18027/management/opportunities/create' });
await wait(`document.querySelectorAll('select').length===7`);
const fill = async (label, value) => {
await evaluate(`(()=>{const label=[...document.querySelectorAll('label')].find(l=>l.textContent.startsWith(${JSON.stringify(label)}));const input=label.querySelector('input');input.focus();input.select();})()`);
if (value === '') {
await send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Backspace', code: 'Backspace', windowsVirtualKeyCode: 8 });
await send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Backspace', code: 'Backspace', windowsVirtualKeyCode: 8 });
} else await send('Input.insertText', { text: value });
};
await fill('Referência', token); await fill('Título', token); await fill('Slug', token);
await select(4, await evaluate(`document.querySelectorAll('select')[4].options[1].value`));
for (let level = 0; level < 4; level++) {
await wait(`!document.querySelectorAll('select')[${level}].disabled && document.querySelectorAll('select')[${level}].options.length>1`);
await select(level, `10000000-0000-4000-8000-${String(level+2).padStart(12, '0')}`);
}
const save = () => evaluate(`[...document.querySelectorAll('button')].find(b=>b.textContent==='Guardar').click()`);
const reopen = async () => {
await wait(`location.pathname==='/management/opportunities' && document.body.innerText.includes(${JSON.stringify(token)})`);
await evaluate(`[...document.querySelectorAll('article')].find(a=>a.textContent.includes(${JSON.stringify(token)})).querySelector('a').click()`);
await wait(`location.pathname.endsWith('/edit') && document.querySelectorAll('select').length===7`);
};
await save(); await reopen();
await wait(`document.querySelectorAll('select')[3].value==='10000000-0000-4000-8000-000000000005'`);
observations.push('create_save_reopen_full_path');
// Server validation must retain the chain and expose the failing field.
await fill('Referência', ''); await save();
await wait(`document.querySelector('[role="alert"]')!==null`);
if (!(await evaluate(`document.querySelectorAll('select')[3].value==='10000000-0000-4000-8000-000000000005'`))) throw Error('Validation lost selection');
await fill('Referência', token);
observations.push('validation_preserves_selection');
await select(2, '');
await wait(`document.querySelectorAll('select')[3].disabled && document.querySelectorAll('select')[3].value===''`);
await save(); await reopen();
await wait(`document.querySelectorAll('select')[1].value==='10000000-0000-4000-8000-000000000003' && document.querySelectorAll('select')[2].value===''`);
observations.push('edit_save_reopen_and_descendant_reset');
await send('Network.setBlockedURLs', { urls: ['*://127.0.0.1:18027/management/territory-options?type=province*'] });
await evaluate(`document.querySelector('input[aria-label="Pesquisar Província"]').focus()`);
await send('Input.insertText', { text: 'Synthetic' });
await wait(`document.body.innerText.includes('Tentar novamente')`);
await send('Network.setBlockedURLs', { urls: [] });
await evaluate(`[...document.querySelectorAll('button')].find(b=>b.textContent==='Tentar novamente').click()`);
await wait(`!document.body.innerText.includes('Tentar novamente') && document.querySelectorAll('select')[0].options.length>1 && !document.querySelectorAll('select')[0].disabled`);
observations.push('network_failure_and_retry');
await evaluate(`document.querySelector('input[aria-label="Pesquisar Distrito"]').focus()`);
await send('Input.insertText', { text: 'no-such-synthetic-district' });
await wait(`document.body.innerText.includes('Sem opções para esta pesquisa.')`);
observations.push('empty_search');
await select(0, '');
await select(0, '10000000-0000-4000-8000-000000000002');
await select(0, '');
await new Promise(resolve => setTimeout(resolve, 500));
if (!(await evaluate(`[...document.querySelectorAll('select')].slice(1,4).every(s=>s.disabled && s.value==='' && s.options.length===1)`))) throw Error('Stale children after rapid parent changes');
observations.push('rapid_parent_change_cancels_and_clears_children');
const widths = [];
for (const width of [390, 1440]) { await send('Emulation.setDeviceMetricsOverride', { width, height: 900, deviceScaleFactor: 1, mobile: width < 600 }); widths.push({ width, noOverflow: await evaluate('document.documentElement.scrollWidth<=innerWidth+1') }); }
if (errors.length || widths.some(w => !w.noOverflow)) throw Error('Browser validation failed');
const report = { date: new Date().toISOString(), observations, requests, responses, widths, javascript_errors: errors.length, scope: 'Synthetic isolated QA' };
fs.writeFileSync('docs/investmoz/gis/opportunity-dropdown-browser.json', JSON.stringify(report, null, 2) + '\n');
console.log(JSON.stringify(report));
} catch (error) {
console.log(JSON.stringify({ requests, responses, responseBodies }));
console.log(await evaluate(`({path:location.pathname,body:document.body.innerText.slice(0,800)})`));
throw error;
} finally { ws.close(); await fetch(`http://127.0.0.1:9224/json/close/${target.id}`); }
+47
View File
@@ -0,0 +1,47 @@
<?php
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\DB;
// Read a pinned, inspected package. Execute only in the isolated QA database.
require __DIR__.'/../vendor/autoload.php';
$app = require __DIR__.'/../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
$db = DB::connection();
if (! $app->environment('testing') || $db->getDatabaseName() !== 'portal_services_test' || (string) $db->getConfig('port') !== '55441') {
throw new RuntimeException('Isolated QA database required.');
}
$package = json_decode(file_get_contents(storage_path('app/private/cartography-sql-package.json')), true, 512, JSON_THROW_ON_ERROR);
$source = 'C:/Mac/Home/Documents/TEMPO/Portal Tempo/Servicos Tempo/prompt/cartografia_moz.xlsx';
if (hash_file('sha256', $source) !== $package['report']['source_sha256'] || $package['report']['source_sha256'] !== '1e36c769c4767420b10f1ff5ee0c2955cdb1b03c98179c5cad549e832ffb1f15') {
throw new RuntimeException('Source checksum mismatch.');
}
$results = [];
for ($run = 1; $run <= 2; $run++) {
$results[] = $db->transaction(function () use ($db, $package, $run) {
foreach ($package['ddl'] as $sql) {
if (! str_starts_with($sql, '--')) {
$db->unprepared($sql);
}
}
foreach ($package['statements'] as $sql) {
// Laravel owns the single transaction covering DDL and all INSERTs.
if (! in_array($sql, ['BEGIN;', 'COMMIT;'], true)) {
$db->unprepared($sql);
}
}
$counts = [];
foreach ($package['report']['counts'] as $table => $expected) {
$counts[$table] = $db->table('investmoz.'.$table)->count();
if ($counts[$table] !== $expected) {
throw new RuntimeException('Reconciliation failed: '.$table);
}
}
$db->statement('SET search_path TO public');
return ['run' => $run, 'counts' => $counts, 'total' => array_sum($counts)];
});
}
$report = ['scope' => 'isolated QA only; source staging tables, not application master data', 'results' => $results];
file_put_contents(base_path('docs/investmoz/gis/sql-workbook-validation.json'), json_encode($report, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR).PHP_EOL);
echo json_encode($report, JSON_THROW_ON_ERROR).PHP_EOL;