const fs = require('node:fs'); const crypto = require('node:crypto'); const dir = 'storage/app/private/investmoz-v2/'; const read = (p) => JSON.parse(fs.readFileSync(dir + p, 'utf8').replace(/^\uFEFF/, '')); const inventory = read('legal-journey-inventory.json'); const active = read('document-reconciliation-active.json'); const documents = active.tables.documents; const resources = active.tables.portal_resources; const records = []; for (const item of inventory.entries) { records.push({ identifier: item.path ?? item.root, origin: { inventoryPath: item.path, status: 'local_provenance_not_official_validation', }, availability: item.status ?? 'inventoried', failure: item.reason ?? null, sha256: item.sha256 ?? null, version: 'not_identified_from_bytes', consultationAuthorization: 'not_determined_by_inventory', legalStatus: 'not_determined_by_inventory', calculationUse: 'not_activated_by_inventory', registeredDocuments: documents .filter( (d) => d.sha256?.toLowerCase() === item.sha256?.toLowerCase(), ) .map((d) => ({ metadata: d, resources: resources .filter((r) => r.document_id === d.id) .map((r) => ({ ...r, links: active.tables.portal_resource_links.filter( (l) => l.resource_id === r.id, ), })), })), }); for (const attachment of item.attachments ?? []) records.push({ identifier: item.path + '::' + attachment.identifier, origin: { parent: item.path, embeddedIdentifier: attachment.identifier, }, sha256: attachment.sha256, availability: 'inventoried_embedded', version: 'not_identified', consultationAuthorization: 'not_determined', legalStatus: 'not_determined', calculationUse: 'not_activated_by_inventory', }); } const groups = new Map(); for (const r of records) { if (!r.sha256) continue; const key = r.sha256.toLowerCase(); groups.set(key, [...(groups.get(key) ?? []), r.identifier]); } for (const r of records) r.byteDuplicates = r.sha256 ? (groups.get(r.sha256.toLowerCase()) ?? []).filter( (p) => p !== r.identifier, ) : []; fs.writeFileSync( dir + 'document-reconciliation-register.json', JSON.stringify( { checked_at: new Date().toISOString(), activeReadOnly: active.target, records, }, null, 2, ), ); function csv(text) { let rows = [], row = [], field = '', quoted = false; for (let i = 0; i < text.length; i++) { let c = text[i]; if (c === '"') { if (quoted && text[i + 1] === '"') { field += '"'; i++; } else quoted = !quoted; } else if (c === ',' && !quoted) { row.push(field); field = ''; } else if (c === '\n' && !quoted) { row.push(field.replace(/\r$/, '')); rows.push(row); row = []; field = ''; } else field += c; } if (field || row.length) { row.push(field); rows.push(row); } return rows; } const index = csv( fs.readFileSync('docs/investmoz/v2/indice-documento-mestre.csv', 'utf8'), ); const paragraphs = read('master-paragraphs.json'); const headings = new Map(); for (const row of index.slice(1)) { const title = row[5]; if (!title) continue; const start = Number(row[2]), end = Number(row[3]); const body = paragraphs .filter((p) => p.p >= start && p.p <= end) .map((p) => p.text.replace(/\s+/g, ' ').trim()) .join('\n'); const key = title.trim().toLowerCase(); headings.set(key, [ ...(headings.get(key) ?? []), { title, start, end, paragraphs: end - start + 1, contentSha256: crypto .createHash('sha256') .update(body) .digest('hex'), }, ]); } const repeated = [...headings.values()] .filter((g) => g.length > 1) .map((g) => ({ title: g[0].title, occurrences: g, comparison: new Set(g.map((r) => r.contentSha256)).size > 1 ? 'historical_content_differs' : 'historical_content_identical', decision: 'pending_human_reconciliation_no_requirement_discarded', })); fs.writeFileSync( 'docs/investmoz/v2/repeticoes-reconciliacao.json', JSON.stringify( { sourceSha256: index[1][1], basis: 'historical_saved_extraction_not_current_locked_original', repeated, }, null, 2, ), ); console.log( JSON.stringify({ records: records.length, duplicateGroups: [...groups.values()].filter((g) => g.length > 1) .length, repeatedGroups: repeated.length, differentGroups: repeated.filter( (g) => g.comparison === 'historical_content_differs', ).length, }), );