"""Inventory the public legacy graph. Never emit HTML, cookies or CSRF values."""
import concurrent.futures
import hashlib
import html
import json
import re
import time
import urllib.parse
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'storage/app/private/investmoz-v2/portal-audit/legacy-discovery'
OUT.mkdir(parents=True, exist_ok=True)
BASE = 'https://investemoz.tempo.co.mz'
HEADERS = {'User-Agent': 'Mozilla/5.0 InvestMoz-QA-content-inventory', 'Accept': 'text/html,application/json,image/*,*/*;q=0.8'}


def fetch(url):
    record = {'url': url, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
    try:
        with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=35) as response:
            data = response.read(20 * 1024 * 1024 + 1)
            if len(data) > 20 * 1024 * 1024:
                raise ValueError('Inventory per-file size limit exceeded')
            mime = response.headers.get_content_type()
            record.update(status=response.status, final_url=response.url, media_type=mime, bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
        suffix = Path(urllib.parse.urlsplit(url).path).suffix or ('.html' if mime == 'text/html' else '.bin')
        path = OUT / (record['sha256'] + suffix)
        path.write_bytes(data)
        record['private_file'] = str(path.relative_to(ROOT)).replace('\\', '/')
        if mime == 'text/html':
            text = data.decode('utf-8', errors='replace')
            match = re.search(r'data-page="([^"]+)"', text)
            if match:
                page = json.loads(html.unescape(match.group(1)))
                # Only public application content, not shared authentication/session props.
                record['component'] = page.get('component')
                public = {k: v for k, v in page.get('props', {}).items() if k not in ['auth', 'errors', 'flash', 'csrf_token', 'ziggy', 'csrf']}
                record['public_props'] = public
            title = re.search(r'<title[^>]*>(.*?)</title>', text, re.S)
            record['title'] = html.unescape(title.group(1)) if title else None
            record['references'] = sorted(set(html.unescape(x) for x in re.findall(r'(?:href|src)=[\"\']([^\"\']+)', text)))
        elif mime in ['text/css', 'application/javascript', 'text/javascript']:
            text = data.decode('utf-8', errors='replace')
            record['references'] = sorted(set(re.findall(r'(?:[\"\'])(/(?:images|storage|uploads|build)/[^\"\'\s]+)[\"\']', text)))
    except Exception as error:
        record.update(status=getattr(error, 'code', None), error=type(error).__name__)
    return record


def urls(value):
    if isinstance(value, dict):
        for item in value.values():
            yield from urls(item)
    elif isinstance(value, list):
        for item in value:
            yield from urls(item)
    elif isinstance(value, str):
        for match in re.findall(r'(?:https?://[^\s<>"\']+|/(?:storage|images|uploads|build)/[^\s<>"\']+)', html.unescape(value)):
            yield match
        yield from re.findall(r'(?:href|src)=[\"\']([^\"\']+)', html.unescape(value))


pending = {BASE + '/', BASE + '/portal-do-investidor', BASE + '/agronegocio', BASE + '/robots.txt', BASE + '/sitemap.xml'}
records = {}
for depth in range(5):
    batch = sorted(pending - records.keys())
    if not batch:
        break
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
        for record in pool.map(fetch, batch):
            records[record['url']] = record
    pending = set()
    for record in records.values():
        refs = list(record.get('references', [])) + list(urls(record.get('public_props', {})))
        props = record.get('public_props', {})
        # The legacy exposes public articles as slugs, not necessarily HTML links in the SPA shell.
        def slugs(value):
            if isinstance(value, dict):
                if isinstance(value.get('slug'), str):
                    yield '/' + value['slug'].lstrip('/')
                for item in value.values():
                    yield from slugs(item)
            elif isinstance(value, list):
                for item in value:
                    yield from slugs(item)
        refs += list(slugs(props))
        for ref in refs:
            url = urllib.parse.urljoin(record['url'], ref)
            parsed = urllib.parse.urlsplit(url)
            is_media = parsed.netloc == 'contentmanager.tempo.co.mz' and parsed.path.startswith('/wp-content/uploads/')
            if (parsed.netloc != 'investemoz.tempo.co.mz' and not is_media) or parsed.query or parsed.fragment or re.search(r'/(?:login|register|logout|admin|dashboard|search|pesquisa|api)(?:/|$)', parsed.path):
                continue
            pending.add(url)
    print(json.dumps({'depth': depth, 'inventoried': len(records), 'remaining_discovered': len(pending - records.keys())}), flush=True)

inventory = {'source': BASE, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'records': list(records.values()), 'unfetched_discovered': sorted(pending - records.keys()), 'note': 'Accessible anonymous graph; query/fragment and authentication endpoints excluded. Raw snapshots remain private and are never served.'}
(OUT / 'inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps({'records': len(records), 'ok': sum(r.get('status') == 200 for r in records.values()), 'html': sum(r.get('media_type') == 'text/html' for r in records.values()), 'pending': len(inventory['unfetched_discovered'])}))