186 lines
11 KiB
Python
186 lines
11 KiB
Python
"""Reconcile observed sector sources and stage originals privately; never write a database."""
|
|
import csv
|
|
import hashlib
|
|
import html
|
|
import json
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import urllib.parse
|
|
from collections import Counter
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DOC = ROOT / 'docs/investmoz/v2/auditoria-portal'
|
|
PRIVATE = ROOT / 'storage/app/private/investmoz-v2/portal-audit'
|
|
OUT = PRIVATE / 'legacy-sector-reconciliation-20261010'
|
|
STAGE = ROOT / 'storage/app/private/document-import-staging/legacy-setores-20261010'
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
STAGE.mkdir(parents=True, exist_ok=True)
|
|
csv.field_size_limit(5_000_000)
|
|
|
|
|
|
def load_csv(name):
|
|
return list(csv.DictReader((DOC / name).open(encoding='utf-8-sig', newline='')))
|
|
|
|
|
|
class Markup(HTMLParser):
|
|
def __init__(self, text):
|
|
super().__init__(convert_charrefs=True)
|
|
self.tags, self.links, self.text = Counter(), [], []
|
|
self.feed(text)
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
self.tags[tag] += 1
|
|
for key, value in attrs:
|
|
if key in ['href', 'src'] and value:
|
|
self.links.append(value)
|
|
|
|
def handle_data(self, data):
|
|
self.text.append(data)
|
|
|
|
|
|
inventory_file = PRIVATE / 'legacy-discovery/inventory.json'
|
|
inventory = json.loads(inventory_file.read_text(encoding='utf-8'))
|
|
records = {r['url']: r for r in inventory['records']}
|
|
# Inventory every existing captured file before selecting this increment.
|
|
for record in records.values():
|
|
if record.get('status') == 200:
|
|
path = ROOT / record['private_file']
|
|
assert path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == record['sha256']
|
|
|
|
home = records['https://investemoz.tempo.co.mz/']['public_props']
|
|
sector_cards = home['dynamicOpportunities']
|
|
assert len(sector_cards) == 5
|
|
remaining = [card for card in sector_cards if card['slug'] != 'agronegocio']
|
|
|
|
# Use the existing anonymous collector for current sources, without rerunning its crawl.
|
|
namespace = {'__file__': str(ROOT / 'tools/legacy-content-discovery.py')}
|
|
collector = (ROOT / 'tools/legacy-content-discovery.py').read_text(encoding='utf-8')
|
|
exec(collector[:collector.index('\npending = ')], namespace)
|
|
fresh = [namespace['fetch'](card['href']) for card in remaining]
|
|
(OUT / 'current-public-captures.json').write_text(json.dumps(fresh, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
assert all(r.get('status') == 200 and r.get('public_props', {}).get('post') for r in fresh), 'Current sector source unavailable; preserve old captures.'
|
|
current = {r['url']: r for r in fresh}
|
|
|
|
preservation = json.loads((PRIVATE / 'qa-readable-preservation.json').read_text(encoding='utf-8'))
|
|
saved_root = Path(preservation['directory'])
|
|
saved = json.loads((saved_root / 'manifest.json').read_text(encoding='utf-8'))
|
|
|
|
|
|
def saved_table(name):
|
|
proof = saved['tables']['"public"."' + name + '"']
|
|
data = (saved_root / proof['file']).read_bytes()
|
|
assert hashlib.sha256(data).hexdigest() == proof['sha256']
|
|
return [json.loads(line) for line in data.splitlines()]
|
|
|
|
|
|
masters = saved_table('sectors')
|
|
library = saved_table('documents')
|
|
resources = saved_table('portal_resources')
|
|
|
|
|
|
def normalized(value):
|
|
import unicodedata
|
|
return re.sub(r'\s+', ' ', ''.join(c for c in unicodedata.normalize('NFD', value.replace('&', ' e ')) if not unicodedata.combining(c))).strip().casefold()
|
|
|
|
|
|
selected, summaries = {}, []
|
|
for card in remaining:
|
|
record = current[card['href']]
|
|
post = record['public_props']['post']
|
|
markup = Markup(post['content'])
|
|
matches = [s for s in masters if normalized(s['name']) == normalized(post['title']) and s['is_active']]
|
|
assert len(matches) == 1, 'Sector master correspondence must be unambiguous.'
|
|
selected[record['url']] = record
|
|
linked = []
|
|
for ref in markup.links:
|
|
url = urllib.parse.urljoin(record['url'], ref)
|
|
child = records.get(url)
|
|
if child and child.get('public_props', {}).get('post'):
|
|
selected[url] = child
|
|
linked.append({'url': url, 'legacy_id': child['public_props']['post']['id'], 'title': child['public_props']['post']['title']})
|
|
summaries.append({'slug': card['slug'], 'title': post['title'], 'url': record['url'], 'legacy_id': post['id'],
|
|
'master_sector': {k: matches[0][k] for k in ['id', 'name', 'slug']},
|
|
'characters': len(re.sub(r'\s+', ' ', ' '.join(markup.text)).strip()), 'linked_articles': linked,
|
|
'tags': dict(markup.tags), 'body_sha256': hashlib.sha256(post['content'].encode()).hexdigest(),
|
|
'same_body_as_initial_capture': post['content'] == records[record['url']]['public_props']['post']['content'],
|
|
'same_modified_as_initial_capture': record['public_props']['seo'].get('modifiedTime') == records[record['url']]['public_props']['seo'].get('modifiedTime')})
|
|
|
|
versions, images, manifest, article_details = {}, {}, [], []
|
|
for record in selected.values():
|
|
post, seo = record['public_props']['post'], record['public_props']['seo']
|
|
body_hash = hashlib.sha256(post['content'].encode()).hexdigest()
|
|
key = (str(post['id']), seo.get('modifiedTime'), body_hash)
|
|
if key in versions:
|
|
versions[key]['urls'].append(record['url'])
|
|
continue
|
|
image_url = urllib.parse.urljoin(record['url'], post['thumbnail'])
|
|
image = records[image_url]
|
|
assert image['status'] == 200 and image['media_type'] == 'image/png'
|
|
images.setdefault(image['sha256'], image)
|
|
content = {'format': 'investmoz-legacy-article-v1', 'source_url': record['url'], 'legacy_id': post['id'],
|
|
'slug': post['slug'], 'title': post['title'], 'author': post['author'], 'legacy_category': post['category'],
|
|
'published_at': seo['publishedTime'], 'modified_at': seo['modifiedTime'], 'content_html': post['content'],
|
|
'image_source_url': image_url, 'image_sha256': image['sha256'], 'references': Markup(post['content']).links}
|
|
file_name = f"post-{post['id']}-{seo['modifiedTime'][:10].replace('-', '')}.txt"
|
|
data = json.dumps(content, ensure_ascii=False, indent=2).encode()
|
|
(STAGE / file_name).write_bytes(data)
|
|
identity = f"legacy-tempo-post-{post['id']}-{seo['modifiedTime'][:10].replace('-', '')}"
|
|
duplicate = next((d['id'] for d in library if d['sha256'].strip() == hashlib.sha256(data).hexdigest()), None)
|
|
conflicts = [d['id'] for d in library if d.get('external_id') == identity and d['sha256'].strip() != hashlib.sha256(data).hexdigest()]
|
|
topic = next((s['master_sector']['slug'] for s in summaries if s['url'] == record['url']), '')
|
|
if post['category'] == 'Turismo':
|
|
topic = 'turismo' # Original category, not a territorial geometry association.
|
|
if record['url'].rstrip('/').endswith('/energia'):
|
|
topic = 'energia-especifica'
|
|
if record['url'].rstrip('/').endswith('/mineracao'):
|
|
topic = 'mineracao'
|
|
manifest.append({'external_id': identity, 'relative_path': STAGE.name + '/' + file_name, 'file_name': file_name,
|
|
'title': post['title'], 'issuer': post['author'], 'document_date': seo['publishedTime'][:10],
|
|
'version_label': 'Legado modificado ' + seo['modifiedTime'], 'source_url': record['url'],
|
|
'source_reference': f"Tempo; post {post['id']}; categoria original {post['category']}; publicado {seo['publishedTime']}; modificado {seo['modifiedTime']}. Conteúdo editorial preservado.",
|
|
'category': 'education', 'topic': topic, 'document_type': 'guide' if post['content'].strip() else 'reference_data',
|
|
'language': 'pt', 'description': 'Exemplar editorial do legado; divulgação separada. ' + ('Corpo vazio no original; não substituir por texto inventado.' if not post['content'].strip() else 'Não representa uma oportunidade concreta ou aprovação externa.')})
|
|
details = {'id': post['id'], 'title': post['title'], 'category': post['category'], 'urls': [record['url']],
|
|
'published': seo['publishedTime'], 'modified': seo['modifiedTime'], 'body_sha256': body_hash,
|
|
'library_bytes_sha256': hashlib.sha256(data).hexdigest(), 'duplicate_document': duplicate, 'identity_conflicts': conflicts,
|
|
'file': file_name, 'tags': dict(Markup(post['content']).tags)}
|
|
versions[key] = details
|
|
article_details.append(details)
|
|
|
|
for image in images.values():
|
|
name = 'image-' + image['sha256'][:16] + '.png'
|
|
shutil.copyfile(ROOT / image['private_file'], STAGE / name)
|
|
assert hashlib.sha256((STAGE / name).read_bytes()).hexdigest() == image['sha256']
|
|
manifest.append({'external_id': 'legacy-tempo-image-' + image['sha256'][:16], 'relative_path': STAGE.name + '/' + name,
|
|
'file_name': name, 'title': 'Imagem original — ' + Path(urllib.parse.urlsplit(image['url']).path).name,
|
|
'issuer': 'Tempo (alojamento; autoria individual não indicada)', 'version_label': image['sha256'][:32],
|
|
'source_url': image['url'], 'source_reference': 'Imagem explicitamente referenciada no thumbnail legado; criação e autoria individual não indicadas.',
|
|
'category': 'media', 'topic': '', 'document_type': 'reference_data', 'language': 'pt',
|
|
'description': 'Original preservado. Fundo genérico não identifica um local, parcela ou oportunidade.'})
|
|
(STAGE / 'manifest.json').write_text(json.dumps({'documents': manifest}, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
|
|
inventory_rows = load_csv('legado-inventario.csv')
|
|
content_rows = load_csv('legado-conteudos-reconciliados.csv')
|
|
for row in inventory_rows:
|
|
if row['URL'] in selected or row['SHA256'] in images:
|
|
row['Destino proposto'] = 'Biblioteca existente; adapter/componente LegacyContent e portal/legacy; URL original preservado'
|
|
row['Situação'] = 'Reconciliado por identidade/versão/hash; lote privado preparado; não importado; scan/divulgação/UI pendentes; C71 bloqueia aplicação'
|
|
for row in content_rows:
|
|
key = (row['ID legado'], row['Modificado'], row['SHA256 conteúdo'])
|
|
if key in versions:
|
|
row['Situação'] = 'Reconciliado; original preparado na biblioteca existente; não importado; sem recurso QA novo; visual/scan/divulgação pendentes'
|
|
|
|
result = {'database_access': False, 'database_writes': 0, 'active_access': False,
|
|
'master_snapshot_at': preservation['created_at'], 'master_snapshot_target': preservation['target'],
|
|
'initial_inventory_files_verified': len(records), 'initial_inventory_sha256': hashlib.sha256(inventory_file.read_bytes()).hexdigest(),
|
|
'sector_summaries': summaries, 'articles': article_details, 'image_count': len(images), 'manifest_count': len(manifest),
|
|
'manifest_path': str((STAGE / 'manifest.json').relative_to(ROOT)), 'import_applied': False,
|
|
'scan_verified': False, 'public_disclosure_applied': False, 'interface_accepted': False,
|
|
'csv_updates': {'legado-inventario.csv': inventory_rows, 'legado-conteudos-reconciliados.csv': content_rows}}
|
|
(OUT / 'reconciliation.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print(json.dumps({'sectors': len(summaries), 'articles': len(article_details), 'images': len(images), 'manifest_items': len(manifest),
|
|
'unchanged_current_sector_bodies': sum(s['same_body_as_initial_capture'] for s in summaries), 'database_writes': 0, 'import_applied': False}))
|