36 lines
2.5 KiB
Python
36 lines
2.5 KiB
Python
import hashlib, json, pathlib, zipfile, xml.etree.ElementTree as ET
|
|
|
|
root = pathlib.Path(r'C:\Mac\Home\Documents\TEMPO\Portal Tempo\Servicos Tempo\portal do investidor')
|
|
out = pathlib.Path('storage/app/private/anchor-project-reconciliation')
|
|
ns = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
|
|
inventory, extracts = [], []
|
|
for file in sorted(root.rglob('*')):
|
|
if not file.is_file(): continue
|
|
record = {'path':str(file), 'bytes':file.stat().st_size, 'analysis':'not_analysed'}
|
|
try:
|
|
data = file.read_bytes()
|
|
record['sha256'] = hashlib.sha256(data).hexdigest()
|
|
if zipfile.is_zipfile(file):
|
|
with zipfile.ZipFile(file) as archive:
|
|
record['attachments'] = [{'path':e.filename,'bytes':e.file_size,'sha256':hashlib.sha256(archive.read(e)).hexdigest()} for e in archive.infolist() if '/media/' in e.filename or '/embeddings/' in e.filename]
|
|
if file.suffix.lower()=='.docx':
|
|
tree=ET.fromstring(archive.read('word/document.xml'))
|
|
paragraphs=[''.join(p.itertext()) for p in []]
|
|
paragraphs=[''.join(t.text or '' for t in p.findall('.//w:t',ns)) for p in tree.findall('.//w:p',ns)]
|
|
text='\n'.join(paragraphs)
|
|
tables=[[[ '\n'.join(''.join(t.text or '' for t in p.findall('.//w:t',ns)) for p in c.findall('./w:p',ns)) for c in r.findall('./w:tc',ns)] for r in table.findall('./w:tr',ns)] for table in tree.findall('.//w:body/w:tbl',ns)]
|
|
extracts.append({'path':str(file),'sha256':record['sha256'],'paragraphs':paragraphs,'tables':tables})
|
|
record['analysis']='text_extracted; relevance_pending'
|
|
except Exception as error:
|
|
record['error']=str(error)
|
|
inventory.append(record)
|
|
out.mkdir(parents=True,exist_ok=True)
|
|
(out/'source-files.json').write_text(json.dumps(inventory,ensure_ascii=False,indent=2),encoding='utf8')
|
|
(out/'source-texts.json').write_text(json.dumps(extracts,ensure_ascii=False,indent=2),encoding='utf8')
|
|
for doc in extracts:
|
|
if 'Matriz detalhada' in doc['path'] or 'v5 programacao' in doc['path'] or doc['path'].endswith('projectos.docx'):
|
|
print(doc['path'])
|
|
relevant=[(i,p) for i,p in enumerate(doc['paragraphs']) if any(s in p for s in ['A01','A03','A04','A09','6.6','6.1','oportunidades complementares','Oportunidades complementares'])]
|
|
for i,p in relevant[:24]: print(i, p[:450])
|
|
print('Files:',len(inventory),'DOCX:',len(extracts))
|