105 lines
6.6 KiB
Python
105 lines
6.6 KiB
Python
"""Inventory and retain source bytes; extract sections without rewriting the DOCX."""
|
|
from pathlib import Path
|
|
from zipfile import ZipFile, BadZipFile
|
|
from datetime import datetime, timezone
|
|
import hashlib, json, shutil, sys, xml.etree.ElementTree as ET
|
|
|
|
root = Path(__file__).resolve().parent.parent
|
|
private = root / 'storage/app/private/investmoz-v2/portal-audit'
|
|
directory = private / 'segmentation-reconciliation-20261011'
|
|
directory.mkdir(exist_ok=True)
|
|
source = Path(sys.argv[1]).resolve()
|
|
assert source.is_file() and source.suffix.lower() == '.docx'
|
|
w = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
|
|
sha = lambda b: hashlib.sha256(b).hexdigest()
|
|
|
|
# Inventory the accessible source directory and every embedded media/object first.
|
|
inventory = []
|
|
for path in sorted(source.parent.rglob('*')):
|
|
if not path.is_file():
|
|
continue
|
|
data = path.read_bytes()
|
|
item = {'path': str(path), 'bytes': len(data), 'sha256': sha(data), 'attachments': []}
|
|
if path.suffix.lower() == '.docx':
|
|
try:
|
|
with ZipFile(path) as archive:
|
|
for name in archive.namelist():
|
|
if name.startswith(('word/media/', 'word/embeddings/')) and not name.endswith('/'):
|
|
content = archive.read(name)
|
|
item['attachments'].append({'part': name, 'bytes': len(content), 'sha256': sha(content)})
|
|
except BadZipFile:
|
|
item['read_error'] = 'Not a valid DOCX ZIP; inventoried without discarding.'
|
|
inventory.append(item)
|
|
|
|
original = source.read_bytes()
|
|
digest = sha(original)
|
|
retained = directory / 'originals' / (digest + '.docx')
|
|
retained.parent.mkdir(exist_ok=True)
|
|
if not retained.exists():
|
|
shutil.copy2(source, retained)
|
|
assert sha(retained.read_bytes()) == digest
|
|
with ZipFile(retained) as archive:
|
|
main = ET.fromstring(archive.read('word/document.xml'))
|
|
rels = {n.get('Id'): n.get('Target') for n in ET.fromstring(archive.read('word/_rels/document.xml.rels'))}
|
|
body = main.find(w + 'body')
|
|
paragraphs, sections = [], []
|
|
section = {'key': 'front', 'heading': 'Frontispício', 'start': 1, 'paragraphs': []}
|
|
sections.append(section)
|
|
table_numbers = {}
|
|
for number, table in enumerate(main.iter(w + 'tbl'), 1):
|
|
for row_no, row in enumerate(table.findall(w + 'tr'), 1):
|
|
for cell_no, cell in enumerate(row.findall(w + 'tc'), 1):
|
|
for p in cell.iter(w + 'p'):
|
|
table_numbers[id(p)] = {'table': number, 'row': row_no, 'cell': cell_no}
|
|
for number, p in enumerate(main.iter(w + 'p'), 1):
|
|
style = p.find(w + 'pPr/' + w + 'pStyle')
|
|
style = style.get(w + 'val', '') if style is not None else ''
|
|
text = ''.join(n.text or '' for n in p.iter(w + 't'))
|
|
deleted = ''.join(n.text or '' for n in p.iter(w + 'delText'))
|
|
if style.startswith(('Ttulo', 'Heading')) and text:
|
|
key = text.split(' ', 1)[0].rstrip('.') if text[0].isdigit() else text
|
|
section = {'key': key, 'heading': text, 'start': number, 'paragraphs': []}
|
|
sections.append(section)
|
|
item = {'paragraph': number, 'style': style, 'text': text,
|
|
'deleted_text_preserved': deleted, 'section': section['key'],
|
|
'locator': 'word/document.xml#p' + str(number),
|
|
'hyperlinks': [{'id': n.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id'),
|
|
'url': rels.get(n.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id')),
|
|
'text': ''.join(t.text or '' for t in n.iter(w + 't'))}
|
|
for n in p.iter(w + 'hyperlink')], **table_numbers.get(id(p), {})}
|
|
paragraphs.append(item)
|
|
section['paragraphs'].append(number)
|
|
properties = {n.tag.rsplit('}', 1)[-1]: n.text for n in ET.fromstring(archive.read('docProps/core.xml'))}
|
|
links = []
|
|
for name in archive.namelist():
|
|
if name.endswith('.rels'):
|
|
for n in ET.fromstring(archive.read(name)):
|
|
if n.get('TargetMode') == 'External':
|
|
links.append({'part': name, 'id': n.get('Id'), 'target': n.get('Target'), 'type': n.get('Type')})
|
|
extra_parts = {}
|
|
for name in archive.namelist():
|
|
if name.startswith(('word/header', 'word/footer', 'word/footnotes', 'word/endnotes', 'word/comments')) and name.endswith('.xml'):
|
|
tree = ET.fromstring(archive.read(name))
|
|
extra_parts[name] = [''.join(n.text or '' for n in p.iter(w + 't')) for p in tree.iter(w + 'p')]
|
|
extraction = {'source': str(source), 'original_sha256': digest, 'bytes': len(original),
|
|
'properties': properties, 'paragraphs': paragraphs, 'sections': sections,
|
|
'tables': len(list(main.iter(w + 'tbl'))), 'external_relationships': links, 'extra_parts': extra_parts,
|
|
'tracked_insertions': len(list(main.iter(w + 'ins'))), 'tracked_deletions': len(list(main.iter(w + 'del'))),
|
|
'page_references_not_inferred': True, 'original_rewritten': False}
|
|
|
|
provenance = {'at': datetime.now(timezone.utc).isoformat(), 'expected_filename': 'segmentar processos v1(3).docx',
|
|
'observed_filename': source.name, 'exact_requested_exemplar_confirmed': source.name == 'segmentar processos v1(3).docx',
|
|
'source': str(source), 'retained': str(retained), 'sha256': digest, 'bytes': len(original),
|
|
'version_properties': properties, 'official_source': False, 'document_kind': 'Project specification',
|
|
'consultation': 'Retained privately for the authorized QA reconciliation',
|
|
'legal_status': 'Programming specification; no law or external decision inferred',
|
|
'rule_activation': False, 'publication': False, 'active_writes': 0,
|
|
'source_directory_inventory': inventory, 'original_attachments': next(x['attachments'] for x in inventory if x['path'] == str(source))}
|
|
for name, value in [('source-provenance.json', provenance), ('extraction.json', extraction)]:
|
|
(directory / name).write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
text = '\n'.join(f"[p{p['paragraph']} {p['section']}] {p['text']}" for p in paragraphs)
|
|
(directory / 'complete-extraction.txt').write_text(text, encoding='utf-8')
|
|
print(json.dumps({'result': 'source_retained_and_extracted', 'sha256': digest, 'paragraphs': len(paragraphs),
|
|
'sections': len(sections), 'inventory_files': len(inventory), 'exact_requested_exemplar_confirmed': provenance['exact_requested_exemplar_confirmed'],
|
|
'active_writes': 0}, ensure_ascii=False))
|