oportunidades 6
tests / ci (push) Canceled after 0s

This commit is contained in:
2026-10-10 21:17:46 +02:00
parent 01405cbf8a
commit 20971d523f
126 changed files with 44362 additions and 3496 deletions
+100
View File
@@ -0,0 +1,100 @@
"""Inventory the public legacy graph. Never emit HTML, cookies or CSRF values."""
import concurrent.futures
import hashlib
import html
import json
import re
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'storage/app/private/investmoz-v2/portal-audit/legacy-discovery'
OUT.mkdir(parents=True, exist_ok=True)
BASE = 'https://investemoz.tempo.co.mz'
HEADERS = {'User-Agent': 'Mozilla/5.0 InvestMoz-QA-content-inventory', 'Accept': 'text/html,application/json,image/*,*/*;q=0.8'}
def fetch(url):
record = {'url': url, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
try:
with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=35) as response:
data = response.read(20 * 1024 * 1024 + 1)
if len(data) > 20 * 1024 * 1024:
raise ValueError('Inventory per-file size limit exceeded')
mime = response.headers.get_content_type()
record.update(status=response.status, final_url=response.url, media_type=mime, bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
suffix = Path(urllib.parse.urlsplit(url).path).suffix or ('.html' if mime == 'text/html' else '.bin')
path = OUT / (record['sha256'] + suffix)
path.write_bytes(data)
record['private_file'] = str(path.relative_to(ROOT)).replace('\\', '/')
if mime == 'text/html':
text = data.decode('utf-8', errors='replace')
match = re.search(r'data-page="([^"]+)"', text)
if match:
page = json.loads(html.unescape(match.group(1)))
# Only public application content, not shared authentication/session props.
record['component'] = page.get('component')
public = {k: v for k, v in page.get('props', {}).items() if k not in ['auth', 'errors', 'flash', 'csrf_token', 'ziggy', 'csrf']}
record['public_props'] = public
title = re.search(r'<title[^>]*>(.*?)</title>', text, re.S)
record['title'] = html.unescape(title.group(1)) if title else None
record['references'] = sorted(set(html.unescape(x) for x in re.findall(r'(?:href|src)=[\"\']([^\"\']+)', text)))
elif mime in ['text/css', 'application/javascript', 'text/javascript']:
text = data.decode('utf-8', errors='replace')
record['references'] = sorted(set(re.findall(r'(?:[\"\'])(/(?:images|storage|uploads|build)/[^\"\'\s]+)[\"\']', text)))
except Exception as error:
record.update(status=getattr(error, 'code', None), error=type(error).__name__)
return record
def urls(value):
if isinstance(value, dict):
for item in value.values():
yield from urls(item)
elif isinstance(value, list):
for item in value:
yield from urls(item)
elif isinstance(value, str):
for match in re.findall(r'(?:https?://[^\s<>"\']+|/(?:storage|images|uploads|build)/[^\s<>"\']+)', html.unescape(value)):
yield match
yield from re.findall(r'(?:href|src)=[\"\']([^\"\']+)', html.unescape(value))
pending = {BASE + '/', BASE + '/portal-do-investidor', BASE + '/agronegocio', BASE + '/robots.txt', BASE + '/sitemap.xml'}
records = {}
for depth in range(5):
batch = sorted(pending - records.keys())
if not batch:
break
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
for record in pool.map(fetch, batch):
records[record['url']] = record
pending = set()
for record in records.values():
refs = list(record.get('references', [])) + list(urls(record.get('public_props', {})))
props = record.get('public_props', {})
# The legacy exposes public articles as slugs, not necessarily HTML links in the SPA shell.
def slugs(value):
if isinstance(value, dict):
if isinstance(value.get('slug'), str):
yield '/' + value['slug'].lstrip('/')
for item in value.values():
yield from slugs(item)
elif isinstance(value, list):
for item in value:
yield from slugs(item)
refs += list(slugs(props))
for ref in refs:
url = urllib.parse.urljoin(record['url'], ref)
parsed = urllib.parse.urlsplit(url)
is_media = parsed.netloc == 'contentmanager.tempo.co.mz' and parsed.path.startswith('/wp-content/uploads/')
if (parsed.netloc != 'investemoz.tempo.co.mz' and not is_media) or parsed.query or parsed.fragment or re.search(r'/(?:login|register|logout|admin|dashboard|search|pesquisa|api)(?:/|$)', parsed.path):
continue
pending.add(url)
print(json.dumps({'depth': depth, 'inventoried': len(records), 'remaining_discovered': len(pending - records.keys())}), flush=True)
inventory = {'source': BASE, 'captured_at_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'records': list(records.values()), 'unfetched_discovered': sorted(pending - records.keys()), 'note': 'Accessible anonymous graph; query/fragment and authentication endpoints excluded. Raw snapshots remain private and are never served.'}
(OUT / 'inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps({'records': len(records), 'ok': sum(r.get('status') == 200 for r in records.values()), 'html': sum(r.get('media_type') == 'text/html' for r in records.values()), 'pending': len(inventory['unfetched_discovered'])}))