This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
"""Inventory the accessible documentary collection before selecting form sources."""
|
||||
from pathlib import Path
|
||||
import hashlib, json, zipfile
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
roots = [Path(r'C:\Mac\Home\Documents\TEMPO\Portal Tempo\Servicos Tempo'), Path(r'C:\Users\macbook\.codex\attachments')]
|
||||
out = Path('storage/app/private/virtual-desk')
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
inventory = []
|
||||
for root in roots:
|
||||
for path in sorted(root.rglob('*')):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
item = dict(path=str(path), size=len(raw), sha256=hashlib.sha256(raw).hexdigest(), analysis='not_analysed', attachments=[])
|
||||
if zipfile.is_zipfile(path):
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
item['attachments'] = [dict(identifier=n, size=len(archive.read(n)), sha256=hashlib.sha256(archive.read(n)).hexdigest()) for n in archive.namelist() if '/embeddings/' in n or '/media/' in n]
|
||||
inventory.append(item)
|
||||
except OSError as error:
|
||||
inventory.append(dict(path=str(path), availability='unreadable', reason=str(error)))
|
||||
(out/'source-inventory.json').write_text(json.dumps(inventory, ensure_ascii=False, indent=2), encoding='utf8')
|
||||
# Selection starts only after the complete inventory was persisted.
|
||||
selected=[]
|
||||
for item in inventory:
|
||||
path=Path(item['path'])
|
||||
if path.suffix.lower()=='.docx' and not path.name.startswith('~$'):
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
root=ET.fromstring(archive.read('word/document.xml'))
|
||||
text='\n'.join(''.join(p.itertext()) for p in root.findall('.//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t'))
|
||||
if any(word in path.name.lower() for word in ['balcão','balcao','formul','apiex']) or ('APIEX' in text and ('FORMULÁRIO' in text or 'Formulário' in text)):
|
||||
target=out/(item['sha256']+'.txt')
|
||||
target.write_text(text, encoding='utf8')
|
||||
selected.append(dict(**item, extraction=str(target), form_status='source_review_pending'))
|
||||
(out/'selected-sources.json').write_text(json.dumps(selected, ensure_ascii=False, indent=2), encoding='utf8')
|
||||
print(json.dumps(dict(files=len(inventory), selected=[dict(path=i['path'], extraction=i['extraction']) for i in selected]), ensure_ascii=False))
|
||||
Reference in New Issue
Block a user