46 lines
2.5 KiB
Python
46 lines
2.5 KiB
Python
"""Preserve uncollected references and inspect the real public visual assets."""
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tools'))
|
|
# Import helpers without executing the discovery crawl again.
|
|
namespace = {'__file__': str(ROOT / 'tools/legacy-content-discovery.py')}
|
|
source = (ROOT / 'tools/legacy-content-discovery.py').read_text(encoding='utf-8')
|
|
exec(source[:source.index('\npending = ')], namespace)
|
|
private = ROOT / 'storage/app/private/investmoz-v2/portal-audit/legacy-discovery'
|
|
path = private / 'inventory.json'
|
|
inventory = json.loads(path.read_text(encoding='utf-8'))
|
|
records = {r['url']: r for r in inventory['records']}
|
|
visual_url = 'https://investemoz.tempo.co.mz/build/assets/welcome-DyF43GB5.js'
|
|
if visual_url not in records:
|
|
records[visual_url] = namespace['fetch'](visual_url)
|
|
for record in list(records.values()):
|
|
if record.get('media_type') in ['application/javascript', 'text/javascript']:
|
|
text = (ROOT / record['private_file']).read_text(encoding='utf-8')
|
|
for ref in re.findall(r'/(?:images|assets/logos)/[^\"\'\s]+', text):
|
|
url = urllib.parse.urljoin(record['url'], ref)
|
|
if url not in records:
|
|
records[url] = namespace['fetch'](url)
|
|
references = set()
|
|
for record in records.values():
|
|
for ref in list(record.get('references', [])) + list(namespace['urls'](record.get('public_props', {}))):
|
|
url = urllib.parse.urljoin(record['url'], ref)
|
|
if urllib.parse.urlsplit(url).scheme in ['http', 'https']:
|
|
references.add((record['url'], url))
|
|
doc = ROOT / 'docs/investmoz/v2/auditoria-portal/legado-referencias.csv'
|
|
with doc.open('w', encoding='utf-8-sig', newline='') as f:
|
|
w = csv.writer(f)
|
|
w.writerow(['Página de origem', 'Referência', 'Recolha', 'Nota'])
|
|
for origin, url in sorted(references):
|
|
w.writerow([origin, url, records.get(url, {}).get('status', 'não recolhido'), 'Referência observada; não constitui relação territorial/oportunidade ou validação jurídica.'])
|
|
inventory['records'] = list(records.values())
|
|
inventory['reference_count'] = len(references)
|
|
path.write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print(json.dumps({'records': len(records), 'references': len(references), 'failed': [{'url': r['url'], 'status': r.get('status')} for r in records.values() if r.get('status') != 200]}))
|