@@ -0,0 +1,100 @@
|
||||
"""Inventory the public legacy graph. Never emit HTML, cookies or CSRF values."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'storage/app/private/investmoz-v2/portal-audit/legacy-discovery'
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
BASE = 'https://investemoz.tempo.co.mz'
|
||||
HEADERS = {'User-Agent': 'Mozilla/5.0 InvestMoz-QA-content-inventory', 'Accept': 'text/html,application/json,image/*,*/*;q=0.8'}
|
||||
|
||||
|
||||
def fetch(url):
|
||||
record = {'url': url, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=35) as response:
|
||||
data = response.read(20 * 1024 * 1024 + 1)
|
||||
if len(data) > 20 * 1024 * 1024:
|
||||
raise ValueError('Inventory per-file size limit exceeded')
|
||||
mime = response.headers.get_content_type()
|
||||
record.update(status=response.status, final_url=response.url, media_type=mime, bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
||||
suffix = Path(urllib.parse.urlsplit(url).path).suffix or ('.html' if mime == 'text/html' else '.bin')
|
||||
path = OUT / (record['sha256'] + suffix)
|
||||
path.write_bytes(data)
|
||||
record['private_file'] = str(path.relative_to(ROOT)).replace('\\', '/')
|
||||
if mime == 'text/html':
|
||||
text = data.decode('utf-8', errors='replace')
|
||||
match = re.search(r'data-page="([^"]+)"', text)
|
||||
if match:
|
||||
page = json.loads(html.unescape(match.group(1)))
|
||||
# Only public application content, not shared authentication/session props.
|
||||
record['component'] = page.get('component')
|
||||
public = {k: v for k, v in page.get('props', {}).items() if k not in ['auth', 'errors', 'flash', 'csrf_token', 'ziggy', 'csrf']}
|
||||
record['public_props'] = public
|
||||
title = re.search(r'<title[^>]*>(.*?)</title>', text, re.S)
|
||||
record['title'] = html.unescape(title.group(1)) if title else None
|
||||
record['references'] = sorted(set(html.unescape(x) for x in re.findall(r'(?:href|src)=[\"\']([^\"\']+)', text)))
|
||||
elif mime in ['text/css', 'application/javascript', 'text/javascript']:
|
||||
text = data.decode('utf-8', errors='replace')
|
||||
record['references'] = sorted(set(re.findall(r'(?:[\"\'])(/(?:images|storage|uploads|build)/[^\"\'\s]+)[\"\']', text)))
|
||||
except Exception as error:
|
||||
record.update(status=getattr(error, 'code', None), error=type(error).__name__)
|
||||
return record
|
||||
|
||||
|
||||
def urls(value):
|
||||
if isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from urls(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from urls(item)
|
||||
elif isinstance(value, str):
|
||||
for match in re.findall(r'(?:https?://[^\s<>"\']+|/(?:storage|images|uploads|build)/[^\s<>"\']+)', html.unescape(value)):
|
||||
yield match
|
||||
yield from re.findall(r'(?:href|src)=[\"\']([^\"\']+)', html.unescape(value))
|
||||
|
||||
|
||||
pending = {BASE + '/', BASE + '/portal-do-investidor', BASE + '/agronegocio', BASE + '/robots.txt', BASE + '/sitemap.xml'}
|
||||
records = {}
|
||||
for depth in range(5):
|
||||
batch = sorted(pending - records.keys())
|
||||
if not batch:
|
||||
break
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
|
||||
for record in pool.map(fetch, batch):
|
||||
records[record['url']] = record
|
||||
pending = set()
|
||||
for record in records.values():
|
||||
refs = list(record.get('references', [])) + list(urls(record.get('public_props', {})))
|
||||
props = record.get('public_props', {})
|
||||
# The legacy exposes public articles as slugs, not necessarily HTML links in the SPA shell.
|
||||
def slugs(value):
|
||||
if isinstance(value, dict):
|
||||
if isinstance(value.get('slug'), str):
|
||||
yield '/' + value['slug'].lstrip('/')
|
||||
for item in value.values():
|
||||
yield from slugs(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from slugs(item)
|
||||
refs += list(slugs(props))
|
||||
for ref in refs:
|
||||
url = urllib.parse.urljoin(record['url'], ref)
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
is_media = parsed.netloc == 'contentmanager.tempo.co.mz' and parsed.path.startswith('/wp-content/uploads/')
|
||||
if (parsed.netloc != 'investemoz.tempo.co.mz' and not is_media) or parsed.query or parsed.fragment or re.search(r'/(?:login|register|logout|admin|dashboard|search|pesquisa|api)(?:/|$)', parsed.path):
|
||||
continue
|
||||
pending.add(url)
|
||||
print(json.dumps({'depth': depth, 'inventoried': len(records), 'remaining_discovered': len(pending - records.keys())}), flush=True)
|
||||
|
||||
inventory = {'source': BASE, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'records': list(records.values()), 'unfetched_discovered': sorted(pending - records.keys()), 'note': 'Accessible anonymous graph; query/fragment and authentication endpoints excluded. Raw snapshots remain private and are never served.'}
|
||||
(OUT / 'inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps({'records': len(records), 'ok': sum(r.get('status') == 200 for r in records.values()), 'html': sum(r.get('media_type') == 'text/html' for r in records.values()), 'pending': len(inventory['unfetched_discovered'])}))
|
||||
Reference in New Issue
Block a user