57 lines
3.5 KiB
Python
57 lines
3.5 KiB
Python
"""Inventory the accessible documentary collection before selecting form sources."""
|
|
from pathlib import Path
|
|
import hashlib, json, zipfile
|
|
from xml.etree import ElementTree as ET
|
|
|
|
roots = [Path(r'C:\Mac\Home\Documents\TEMPO\Portal Tempo\Servicos Tempo'), Path(r'C:\Users\macbook\.codex\attachments')]
|
|
out = Path('storage/app/private/virtual-desk')
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
inventory = []
|
|
for root in roots:
|
|
for path in sorted(root.rglob('*')):
|
|
if not path.is_file():
|
|
continue
|
|
try:
|
|
raw = path.read_bytes()
|
|
item = dict(path=str(path), size=len(raw), sha256=hashlib.sha256(raw).hexdigest(), analysis='not_analysed', attachments=[])
|
|
if zipfile.is_zipfile(path):
|
|
with zipfile.ZipFile(path) as archive:
|
|
item['attachments'] = [dict(identifier=n, size=len(archive.read(n)), sha256=hashlib.sha256(archive.read(n)).hexdigest()) for n in archive.namelist() if '/embeddings/' in n or '/media/' in n]
|
|
inventory.append(item)
|
|
except OSError as error:
|
|
inventory.append(dict(path=str(path), availability='unreadable', reason=str(error)))
|
|
(out/'source-inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf8')
|
|
# Selection starts only after the complete inventory was persisted.
|
|
selected=[]
|
|
for item in inventory:
|
|
path=Path(item['path'])
|
|
if path.suffix.lower()=='.docx' and not path.name.startswith('~$'):
|
|
with zipfile.ZipFile(path) as archive:
|
|
root=ET.fromstring(archive.read('word/document.xml'))
|
|
text='\n'.join(''.join(p.itertext()) for p in root.findall('.//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t'))
|
|
if any(word in path.name.lower() for word in ['balcão','balcao','formul','apiex']) or ('APIEX' in text and ('FORMULÁRIO' in text or 'Formulário' in text)):
|
|
target=out/(item['sha256']+'.txt')
|
|
target.write_text(text, encoding='utf8')
|
|
selected.append(dict(**item, extraction=str(target), form_status='source_review_pending'))
|
|
(out/'selected-sources.json').write_text(json.dumps(selected, ensure_ascii=False, indent=2), encoding='utf8')
|
|
# This is a private register, never an import or automatic publication. Original
|
|
# bytes remain untouched. Field text is preserved verbatim, not homologated.
|
|
forms=[]
|
|
for item in inventory:
|
|
path=Path(item['path'])
|
|
if path.name.startswith('~$') or not any(x in str(path).lower() for x in ['\\formularios\\', '\\formulario\\']):
|
|
continue
|
|
record=dict(**item, identifier=path.name, edition='not_identified', consultation='local_inventory_only_not_authorized_publication', legal_documentary_status='not_verified', rule_use='none', official_confirmation='pending', fields_status='preserved_in_original_pending_verification')
|
|
if path.suffix.lower()=='.docx':
|
|
with zipfile.ZipFile(path) as archive:
|
|
field_xml=archive.read('word/document.xml')
|
|
field_path=out/(item['sha256']+'-fields.xml')
|
|
field_path.write_bytes(field_xml)
|
|
record['original_fields_xml']=str(field_path)
|
|
record['nature']='design_or_template_source_not_confirmed_official_form'
|
|
else:
|
|
record['nature']='scanned_form_candidate_not_confirmed_official'
|
|
forms.append(record)
|
|
(out/'form-register.json').write_text(json.dumps(forms, ensure_ascii=False, indent=2), encoding='utf8')
|
|
print(json.dumps(dict(files=len(inventory), selected=[dict(path=i['path'], extraction=i['extraction']) for i in selected]), ensure_ascii=False))
|