"""Reconcile observed sector sources and stage originals privately; never write a database."""
import csv
import hashlib
import html
import json
import re
import shutil
import sys
import urllib.parse
from collections import Counter
from html.parser import HTMLParser
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DOC = ROOT / 'docs/investmoz/v2/auditoria-portal'
PRIVATE = ROOT / 'storage/app/private/investmoz-v2/portal-audit'
OUT = PRIVATE / 'legacy-sector-reconciliation-20261010'
STAGE = ROOT / 'storage/app/private/document-import-staging/legacy-setores-20261010'
OUT.mkdir(parents=True, exist_ok=True)
STAGE.mkdir(parents=True, exist_ok=True)
csv.field_size_limit(5_000_000)


def load_csv(name):
    return list(csv.DictReader((DOC / name).open(encoding='utf-8-sig', newline='')))


class Markup(HTMLParser):
    def __init__(self, text):
        super().__init__(convert_charrefs=True)
        self.tags, self.links, self.text = Counter(), [], []
        self.feed(text)

    def handle_starttag(self, tag, attrs):
        self.tags[tag] += 1
        for key, value in attrs:
            if key in ['href', 'src'] and value:
                self.links.append(value)

    def handle_data(self, data):
        self.text.append(data)


inventory_file = PRIVATE / 'legacy-discovery/inventory.json'
inventory = json.loads(inventory_file.read_text(encoding='utf-8'))
records = {r['url']: r for r in inventory['records']}
# Inventory every existing captured file before selecting this increment.
for record in records.values():
    if record.get('status') == 200:
        path = ROOT / record['private_file']
        assert path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == record['sha256']

home = records['https://investemoz.tempo.co.mz/']['public_props']
sector_cards = home['dynamicOpportunities']
assert len(sector_cards) == 5
remaining = [card for card in sector_cards if card['slug'] != 'agronegocio']

# Use the existing anonymous collector for current sources, without rerunning its crawl.
namespace = {'__file__': str(ROOT / 'tools/legacy-content-discovery.py')}
collector = (ROOT / 'tools/legacy-content-discovery.py').read_text(encoding='utf-8')
exec(collector[:collector.index('\npending = ')], namespace)
fresh = [namespace['fetch'](card['href']) for card in remaining]
(OUT / 'current-public-captures.json').write_text(json.dumps(fresh, ensure_ascii=False, indent=2), encoding='utf-8')
assert all(r.get('status') == 200 and r.get('public_props', {}).get('post') for r in fresh), 'Current sector source unavailable; preserve old captures.'
current = {r['url']: r for r in fresh}

preservation = json.loads((PRIVATE / 'qa-readable-preservation.json').read_text(encoding='utf-8'))
saved_root = Path(preservation['directory'])
saved = json.loads((saved_root / 'manifest.json').read_text(encoding='utf-8'))


def saved_table(name):
    proof = saved['tables']['"public"."' + name + '"']
    data = (saved_root / proof['file']).read_bytes()
    assert hashlib.sha256(data).hexdigest() == proof['sha256']
    return [json.loads(line) for line in data.splitlines()]


masters = saved_table('sectors')
library = saved_table('documents')
resources = saved_table('portal_resources')


def normalized(value):
    import unicodedata
    return re.sub(r'\s+', ' ', ''.join(c for c in unicodedata.normalize('NFD', value.replace('&', ' e ')) if not unicodedata.combining(c))).strip().casefold()


selected, summaries = {}, []
for card in remaining:
    record = current[card['href']]
    post = record['public_props']['post']
    markup = Markup(post['content'])
    matches = [s for s in masters if normalized(s['name']) == normalized(post['title']) and s['is_active']]
    assert len(matches) == 1, 'Sector master correspondence must be unambiguous.'
    selected[record['url']] = record
    linked = []
    for ref in markup.links:
        url = urllib.parse.urljoin(record['url'], ref)
        child = records.get(url)
        if child and child.get('public_props', {}).get('post'):
            selected[url] = child
            linked.append({'url': url, 'legacy_id': child['public_props']['post']['id'], 'title': child['public_props']['post']['title']})
    summaries.append({'slug': card['slug'], 'title': post['title'], 'url': record['url'], 'legacy_id': post['id'],
        'master_sector': {k: matches[0][k] for k in ['id', 'name', 'slug']},
        'characters': len(re.sub(r'\s+', ' ', ' '.join(markup.text)).strip()), 'linked_articles': linked,
        'tags': dict(markup.tags), 'body_sha256': hashlib.sha256(post['content'].encode()).hexdigest(),
        'same_body_as_initial_capture': post['content'] == records[record['url']]['public_props']['post']['content'],
        'same_modified_as_initial_capture': record['public_props']['seo'].get('modifiedTime') == records[record['url']]['public_props']['seo'].get('modifiedTime')})

versions, images, manifest, article_details = {}, {}, [], []
for record in selected.values():
    post, seo = record['public_props']['post'], record['public_props']['seo']
    body_hash = hashlib.sha256(post['content'].encode()).hexdigest()
    key = (str(post['id']), seo.get('modifiedTime'), body_hash)
    if key in versions:
        versions[key]['urls'].append(record['url'])
        continue
    image_url = urllib.parse.urljoin(record['url'], post['thumbnail'])
    image = records[image_url]
    assert image['status'] == 200 and image['media_type'] == 'image/png'
    images.setdefault(image['sha256'], image)
    content = {'format': 'investmoz-legacy-article-v1', 'source_url': record['url'], 'legacy_id': post['id'],
        'slug': post['slug'], 'title': post['title'], 'author': post['author'], 'legacy_category': post['category'],
        'published_at': seo['publishedTime'], 'modified_at': seo['modifiedTime'], 'content_html': post['content'],
        'image_source_url': image_url, 'image_sha256': image['sha256'], 'references': Markup(post['content']).links}
    file_name = f"post-{post['id']}-{seo['modifiedTime'][:10].replace('-', '')}.txt"
    data = json.dumps(content, ensure_ascii=False, indent=2).encode()
    (STAGE / file_name).write_bytes(data)
    identity = f"legacy-tempo-post-{post['id']}-{seo['modifiedTime'][:10].replace('-', '')}"
    duplicate = next((d['id'] for d in library if d['sha256'].strip() == hashlib.sha256(data).hexdigest()), None)
    conflicts = [d['id'] for d in library if d.get('external_id') == identity and d['sha256'].strip() != hashlib.sha256(data).hexdigest()]
    topic = next((s['master_sector']['slug'] for s in summaries if s['url'] == record['url']), '')
    if post['category'] == 'Turismo':
        topic = 'turismo'  # Original category, not a territorial geometry association.
    if record['url'].rstrip('/').endswith('/energia'):
        topic = 'energia-especifica'
    if record['url'].rstrip('/').endswith('/mineracao'):
        topic = 'mineracao'
    manifest.append({'external_id': identity, 'relative_path': STAGE.name + '/' + file_name, 'file_name': file_name,
        'title': post['title'], 'issuer': post['author'], 'document_date': seo['publishedTime'][:10],
        'version_label': 'Legado modificado ' + seo['modifiedTime'], 'source_url': record['url'],
        'source_reference': f"Tempo; post {post['id']}; categoria original {post['category']}; publicado {seo['publishedTime']}; modificado {seo['modifiedTime']}. Conteúdo editorial preservado.",
        'category': 'education', 'topic': topic, 'document_type': 'guide' if post['content'].strip() else 'reference_data',
        'language': 'pt', 'description': 'Exemplar editorial do legado; divulgação separada. ' + ('Corpo vazio no original; não substituir por texto inventado.' if not post['content'].strip() else 'Não representa uma oportunidade concreta ou aprovação externa.')})
    details = {'id': post['id'], 'title': post['title'], 'category': post['category'], 'urls': [record['url']],
        'published': seo['publishedTime'], 'modified': seo['modifiedTime'], 'body_sha256': body_hash,
        'library_bytes_sha256': hashlib.sha256(data).hexdigest(), 'duplicate_document': duplicate, 'identity_conflicts': conflicts,
        'file': file_name, 'tags': dict(Markup(post['content']).tags)}
    versions[key] = details
    article_details.append(details)

for image in images.values():
    name = 'image-' + image['sha256'][:16] + '.png'
    shutil.copyfile(ROOT / image['private_file'], STAGE / name)
    assert hashlib.sha256((STAGE / name).read_bytes()).hexdigest() == image['sha256']
    manifest.append({'external_id': 'legacy-tempo-image-' + image['sha256'][:16], 'relative_path': STAGE.name + '/' + name,
        'file_name': name, 'title': 'Imagem original — ' + Path(urllib.parse.urlsplit(image['url']).path).name,
        'issuer': 'Tempo (alojamento; autoria individual não indicada)', 'version_label': image['sha256'][:32],
        'source_url': image['url'], 'source_reference': 'Imagem explicitamente referenciada no thumbnail legado; criação e autoria individual não indicadas.',
        'category': 'media', 'topic': '', 'document_type': 'reference_data', 'language': 'pt',
        'description': 'Original preservado. Fundo genérico não identifica um local, parcela ou oportunidade.'})
(STAGE / 'manifest.json').write_text(json.dumps({'documents': manifest}, ensure_ascii=False, indent=2), encoding='utf-8')

inventory_rows = load_csv('legado-inventario.csv')
content_rows = load_csv('legado-conteudos-reconciliados.csv')
for row in inventory_rows:
    if row['URL'] in selected or row['SHA256'] in images:
        row['Destino proposto'] = 'Biblioteca existente; adapter/componente LegacyContent e portal/legacy; URL original preservado'
        row['Situação'] = 'Reconciliado por identidade/versão/hash; lote privado preparado; não importado; scan/divulgação/UI pendentes; C71 bloqueia aplicação'
for row in content_rows:
    key = (row['ID legado'], row['Modificado'], row['SHA256 conteúdo'])
    if key in versions:
        row['Situação'] = 'Reconciliado; original preparado na biblioteca existente; não importado; sem recurso QA novo; visual/scan/divulgação pendentes'

result = {'database_access': False, 'database_writes': 0, 'active_access': False,
    'master_snapshot_at': preservation['created_at'], 'master_snapshot_target': preservation['target'],
    'initial_inventory_files_verified': len(records), 'initial_inventory_sha256': hashlib.sha256(inventory_file.read_bytes()).hexdigest(),
    'sector_summaries': summaries, 'articles': article_details, 'image_count': len(images), 'manifest_count': len(manifest),
    'manifest_path': str((STAGE / 'manifest.json').relative_to(ROOT)), 'import_applied': False,
    'scan_verified': False, 'public_disclosure_applied': False, 'interface_accepted': False,
    'csv_updates': {'legado-inventario.csv': inventory_rows, 'legado-conteudos-reconciliados.csv': content_rows}}
(OUT / 'reconciliation.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps({'sectors': len(summaries), 'articles': len(article_details), 'images': len(images), 'manifest_items': len(manifest),
    'unchanged_current_sector_bodies': sum(s['same_body_as_initial_capture'] for s in summaries), 'database_writes': 0, 'import_applied': False}))