101 lines
5.5 KiB
Python
101 lines
5.5 KiB
Python
"""Inventory the public legacy graph. Never emit HTML, cookies or CSRF values."""
|
|
import concurrent.futures
|
|
import hashlib
|
|
import html
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / 'storage/app/private/investmoz-v2/portal-audit/legacy-discovery'
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
BASE = 'https://investemoz.tempo.co.mz'
|
|
HEADERS = {'User-Agent': 'Mozilla/5.0 InvestMoz-QA-content-inventory', 'Accept': 'text/html,application/json,image/*,*/*;q=0.8'}
|
|
|
|
|
|
def fetch(url):
|
|
record = {'url': url, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
|
|
try:
|
|
with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=35) as response:
|
|
data = response.read(20 * 1024 * 1024 + 1)
|
|
if len(data) > 20 * 1024 * 1024:
|
|
raise ValueError('Inventory per-file size limit exceeded')
|
|
mime = response.headers.get_content_type()
|
|
record.update(status=response.status, final_url=response.url, media_type=mime, bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
|
suffix = Path(urllib.parse.urlsplit(url).path).suffix or ('.html' if mime == 'text/html' else '.bin')
|
|
path = OUT / (record['sha256'] + suffix)
|
|
path.write_bytes(data)
|
|
record['private_file'] = str(path.relative_to(ROOT)).replace('\\', '/')
|
|
if mime == 'text/html':
|
|
text = data.decode('utf-8', errors='replace')
|
|
match = re.search(r'data-page="([^"]+)"', text)
|
|
if match:
|
|
page = json.loads(html.unescape(match.group(1)))
|
|
# Only public application content, not shared authentication/session props.
|
|
record['component'] = page.get('component')
|
|
public = {k: v for k, v in page.get('props', {}).items() if k not in ['auth', 'errors', 'flash', 'csrf_token', 'ziggy', 'csrf']}
|
|
record['public_props'] = public
|
|
title = re.search(r'<title[^>]*>(.*?)</title>', text, re.S)
|
|
record['title'] = html.unescape(title.group(1)) if title else None
|
|
record['references'] = sorted(set(html.unescape(x) for x in re.findall(r'(?:href|src)=[\"\']([^\"\']+)', text)))
|
|
elif mime in ['text/css', 'application/javascript', 'text/javascript']:
|
|
text = data.decode('utf-8', errors='replace')
|
|
record['references'] = sorted(set(re.findall(r'(?:[\"\'])(/(?:images|storage|uploads|build)/[^\"\'\s]+)[\"\']', text)))
|
|
except Exception as error:
|
|
record.update(status=getattr(error, 'code', None), error=type(error).__name__)
|
|
return record
|
|
|
|
|
|
def urls(value):
|
|
if isinstance(value, dict):
|
|
for item in value.values():
|
|
yield from urls(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
yield from urls(item)
|
|
elif isinstance(value, str):
|
|
for match in re.findall(r'(?:https?://[^\s<>"\']+|/(?:storage|images|uploads|build)/[^\s<>"\']+)', html.unescape(value)):
|
|
yield match
|
|
yield from re.findall(r'(?:href|src)=[\"\']([^\"\']+)', html.unescape(value))
|
|
|
|
|
|
pending = {BASE + '/', BASE + '/portal-do-investidor', BASE + '/agronegocio', BASE + '/robots.txt', BASE + '/sitemap.xml'}
|
|
records = {}
|
|
for depth in range(5):
|
|
batch = sorted(pending - records.keys())
|
|
if not batch:
|
|
break
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
|
|
for record in pool.map(fetch, batch):
|
|
records[record['url']] = record
|
|
pending = set()
|
|
for record in records.values():
|
|
refs = list(record.get('references', [])) + list(urls(record.get('public_props', {})))
|
|
props = record.get('public_props', {})
|
|
# The legacy exposes public articles as slugs, not necessarily HTML links in the SPA shell.
|
|
def slugs(value):
|
|
if isinstance(value, dict):
|
|
if isinstance(value.get('slug'), str):
|
|
yield '/' + value['slug'].lstrip('/')
|
|
for item in value.values():
|
|
yield from slugs(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
yield from slugs(item)
|
|
refs += list(slugs(props))
|
|
for ref in refs:
|
|
url = urllib.parse.urljoin(record['url'], ref)
|
|
parsed = urllib.parse.urlsplit(url)
|
|
is_media = parsed.netloc == 'contentmanager.tempo.co.mz' and parsed.path.startswith('/wp-content/uploads/')
|
|
if (parsed.netloc != 'investemoz.tempo.co.mz' and not is_media) or parsed.query or parsed.fragment or re.search(r'/(?:login|register|logout|admin|dashboard|search|pesquisa|api)(?:/|$)', parsed.path):
|
|
continue
|
|
pending.add(url)
|
|
print(json.dumps({'depth': depth, 'inventoried': len(records), 'remaining_discovered': len(pending - records.keys())}), flush=True)
|
|
|
|
inventory = {'source': BASE, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'records': list(records.values()), 'unfetched_discovered': sorted(pending - records.keys()), 'note': 'Accessible anonymous graph; query/fragment and authentication endpoints excluded. Raw snapshots remain private and are never served.'}
|
|
(OUT / 'inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print(json.dumps({'records': len(records), 'ok': sum(r.get('status') == 200 for r in records.values()), 'html': sum(r.get('media_type') == 'text/html' for r in records.values()), 'pending': len(inventory['unfetched_discovered'])}))
|