38 lines
2.3 KiB
Python
38 lines
2.3 KiB
Python
"""Inventory the accessible documentary collection before selecting form sources."""
|
|
from pathlib import Path
|
|
import hashlib, json, zipfile
|
|
from xml.etree import ElementTree as ET
|
|
|
|
roots = [Path(r'C:\Mac\Home\Documents\TEMPO\Portal Tempo\Servicos Tempo'), Path(r'C:\Users\macbook\.codex\attachments')]
|
|
out = Path('storage/app/private/virtual-desk')
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
inventory = []
|
|
for root in roots:
|
|
for path in sorted(root.rglob('*')):
|
|
if not path.is_file():
|
|
continue
|
|
try:
|
|
raw = path.read_bytes()
|
|
item = dict(path=str(path), size=len(raw), sha256=hashlib.sha256(raw).hexdigest(), analysis='not_analysed', attachments=[])
|
|
if zipfile.is_zipfile(path):
|
|
with zipfile.ZipFile(path) as archive:
|
|
item['attachments'] = [dict(identifier=n, size=len(archive.read(n)), sha256=hashlib.sha256(archive.read(n)).hexdigest()) for n in archive.namelist() if '/embeddings/' in n or '/media/' in n]
|
|
inventory.append(item)
|
|
except OSError as error:
|
|
inventory.append(dict(path=str(path), availability='unreadable', reason=str(error)))
|
|
(out/'source-inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf8')
|
|
# Selection starts only after the complete inventory was persisted.
|
|
selected=[]
|
|
for item in inventory:
|
|
path=Path(item['path'])
|
|
if path.suffix.lower()=='.docx' and not path.name.startswith('~$'):
|
|
with zipfile.ZipFile(path) as archive:
|
|
root=ET.fromstring(archive.read('word/document.xml'))
|
|
text='\n'.join(''.join(p.itertext()) for p in root.findall('.//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t'))
|
|
if any(word in path.name.lower() for word in ['balcão','balcao','formul','apiex']) or ('APIEX' in text and ('FORMULÁRIO' in text or 'Formulário' in text)):
|
|
target=out/(item['sha256']+'.txt')
|
|
target.write_text(text, encoding='utf8')
|
|
selected.append(dict(**item, extraction=str(target), form_status='source_review_pending'))
|
|
(out/'selected-sources.json').write_text(json.dumps(selected, ensure_ascii=False, indent=2), encoding='utf8')
|
|
print(json.dumps(dict(files=len(inventory), selected=[dict(path=i['path'], extraction=i['extraction']) for i in selected]), ensure_ascii=False))
|