Investidor-V2/tools/investmoz-v2-document-reconciliation.cjs

const fs = require('node:fs');
const crypto = require('node:crypto');
const dir = 'storage/app/private/investmoz-v2/';
const read = (p) =>
    JSON.parse(fs.readFileSync(dir + p, 'utf8').replace(/^\uFEFF/, ''));
const inventory = read('legal-journey-inventory.json');
const active = read('document-reconciliation-active.json');
const documents = active.tables.documents;
const resources = active.tables.portal_resources;
const records = [];
for (const item of inventory.entries) {
    records.push({
        identifier: item.path ?? item.root,
        origin: {
            inventoryPath: item.path,
            status: 'local_provenance_not_official_validation',
        },
        availability: item.status ?? 'inventoried',
        failure: item.reason ?? null,
        sha256: item.sha256 ?? null,
        version: 'not_identified_from_bytes',
        consultationAuthorization: 'not_determined_by_inventory',
        legalStatus: 'not_determined_by_inventory',
        calculationUse: 'not_activated_by_inventory',
        registeredDocuments: documents
            .filter(
                (d) => d.sha256?.toLowerCase() === item.sha256?.toLowerCase(),
            )
            .map((d) => ({
                metadata: d,
                resources: resources
                    .filter((r) => r.document_id === d.id)
                    .map((r) => ({
                        ...r,
                        links: active.tables.portal_resource_links.filter(
                            (l) => l.resource_id === r.id,
                        ),
                    })),
            })),
    });
    for (const attachment of item.attachments ?? [])
        records.push({
            identifier: item.path + '::' + attachment.identifier,
            origin: {
                parent: item.path,
                embeddedIdentifier: attachment.identifier,
            },
            sha256: attachment.sha256,
            availability: 'inventoried_embedded',
            version: 'not_identified',
            consultationAuthorization: 'not_determined',
            legalStatus: 'not_determined',
            calculationUse: 'not_activated_by_inventory',
        });
}
const groups = new Map();
for (const r of records) {
    if (!r.sha256) continue;
    const key = r.sha256.toLowerCase();
    groups.set(key, [...(groups.get(key) ?? []), r.identifier]);
}
for (const r of records)
    r.byteDuplicates = r.sha256
        ? (groups.get(r.sha256.toLowerCase()) ?? []).filter(
              (p) => p !== r.identifier,
          )
        : [];
fs.writeFileSync(
    dir + 'document-reconciliation-register.json',
    JSON.stringify(
        {
            checked_at: new Date().toISOString(),
            activeReadOnly: active.target,
            records,
        },
        null,
        2,
    ),
);
function csv(text) {
    let rows = [],
        row = [],
        field = '',
        quoted = false;
    for (let i = 0; i < text.length; i++) {
        let c = text[i];
        if (c === '"') {
            if (quoted && text[i + 1] === '"') {
                field += '"';
                i++;
            } else quoted = !quoted;
        } else if (c === ',' && !quoted) {
            row.push(field);
            field = '';
        } else if (c === '\n' && !quoted) {
            row.push(field.replace(/\r$/, ''));
            rows.push(row);
            row = [];
            field = '';
        } else field += c;
    }
    if (field || row.length) {
        row.push(field);
        rows.push(row);
    }
    return rows;
}
const index = csv(
    fs.readFileSync('docs/investmoz/v2/indice-documento-mestre.csv', 'utf8'),
);
const paragraphs = read('master-paragraphs.json');
const headings = new Map();
for (const row of index.slice(1)) {
    const title = row[5];
    if (!title) continue;
    const start = Number(row[2]),
        end = Number(row[3]);
    const body = paragraphs
        .filter((p) => p.p >= start && p.p <= end)
        .map((p) => p.text.replace(/\s+/g, ' ').trim())
        .join('\n');
    const key = title.trim().toLowerCase();
    headings.set(key, [
        ...(headings.get(key) ?? []),
        {
            title,
            start,
            end,
            paragraphs: end - start + 1,
            contentSha256: crypto
                .createHash('sha256')
                .update(body)
                .digest('hex'),
        },
    ]);
}
const repeated = [...headings.values()]
    .filter((g) => g.length > 1)
    .map((g) => ({
        title: g[0].title,
        occurrences: g,
        comparison:
            new Set(g.map((r) => r.contentSha256)).size > 1
                ? 'historical_content_differs'
                : 'historical_content_identical',
        decision: 'pending_human_reconciliation_no_requirement_discarded',
    }));
fs.writeFileSync(
    'docs/investmoz/v2/repeticoes-reconciliacao.json',
    JSON.stringify(
        {
            sourceSha256: index[1][1],
            basis: 'historical_saved_extraction_not_current_locked_original',
            repeated,
        },
        null,
        2,
    ),
);
console.log(
    JSON.stringify({
        records: records.length,
        duplicateGroups: [...groups.values()].filter((g) => g.length > 1)
            .length,
        repeatedGroups: repeated.length,
        differentGroups: repeated.filter(
            (g) => g.comparison === 'historical_content_differs',
        ).length,
    }),
);