Liga acervo documental e guias a jornada do investidor
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
const fs = require('node:fs');
|
||||
const crypto = require('node:crypto');
|
||||
const dir = 'storage/app/private/investmoz-v2/';
|
||||
const read = (p) =>
|
||||
JSON.parse(fs.readFileSync(dir + p, 'utf8').replace(/^\uFEFF/, ''));
|
||||
const inventory = read('legal-journey-inventory.json');
|
||||
const active = read('document-reconciliation-active.json');
|
||||
const documents = active.tables.documents;
|
||||
const resources = active.tables.portal_resources;
|
||||
const records = [];
|
||||
for (const item of inventory.entries) {
|
||||
records.push({
|
||||
identifier: item.path ?? item.root,
|
||||
origin: {
|
||||
inventoryPath: item.path,
|
||||
status: 'local_provenance_not_official_validation',
|
||||
},
|
||||
availability: item.status ?? 'inventoried',
|
||||
failure: item.reason ?? null,
|
||||
sha256: item.sha256 ?? null,
|
||||
version: 'not_identified_from_bytes',
|
||||
consultationAuthorization: 'not_determined_by_inventory',
|
||||
legalStatus: 'not_determined_by_inventory',
|
||||
calculationUse: 'not_activated_by_inventory',
|
||||
registeredDocuments: documents
|
||||
.filter(
|
||||
(d) => d.sha256?.toLowerCase() === item.sha256?.toLowerCase(),
|
||||
)
|
||||
.map((d) => ({
|
||||
metadata: d,
|
||||
resources: resources
|
||||
.filter((r) => r.document_id === d.id)
|
||||
.map((r) => ({
|
||||
...r,
|
||||
links: active.tables.portal_resource_links.filter(
|
||||
(l) => l.resource_id === r.id,
|
||||
),
|
||||
})),
|
||||
})),
|
||||
});
|
||||
for (const attachment of item.attachments ?? [])
|
||||
records.push({
|
||||
identifier: item.path + '::' + attachment.identifier,
|
||||
origin: {
|
||||
parent: item.path,
|
||||
embeddedIdentifier: attachment.identifier,
|
||||
},
|
||||
sha256: attachment.sha256,
|
||||
availability: 'inventoried_embedded',
|
||||
version: 'not_identified',
|
||||
consultationAuthorization: 'not_determined',
|
||||
legalStatus: 'not_determined',
|
||||
calculationUse: 'not_activated_by_inventory',
|
||||
});
|
||||
}
|
||||
const groups = new Map();
|
||||
for (const r of records) {
|
||||
if (!r.sha256) continue;
|
||||
const key = r.sha256.toLowerCase();
|
||||
groups.set(key, [...(groups.get(key) ?? []), r.identifier]);
|
||||
}
|
||||
for (const r of records)
|
||||
r.byteDuplicates = r.sha256
|
||||
? (groups.get(r.sha256.toLowerCase()) ?? []).filter(
|
||||
(p) => p !== r.identifier,
|
||||
)
|
||||
: [];
|
||||
fs.writeFileSync(
|
||||
dir + 'document-reconciliation-register.json',
|
||||
JSON.stringify(
|
||||
{
|
||||
checked_at: new Date().toISOString(),
|
||||
activeReadOnly: active.target,
|
||||
records,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
function csv(text) {
|
||||
let rows = [],
|
||||
row = [],
|
||||
field = '',
|
||||
quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
let c = text[i];
|
||||
if (c === '"') {
|
||||
if (quoted && text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else quoted = !quoted;
|
||||
} else if (c === ',' && !quoted) {
|
||||
row.push(field);
|
||||
field = '';
|
||||
} else if (c === '\n' && !quoted) {
|
||||
row.push(field.replace(/\r$/, ''));
|
||||
rows.push(row);
|
||||
row = [];
|
||||
field = '';
|
||||
} else field += c;
|
||||
}
|
||||
if (field || row.length) {
|
||||
row.push(field);
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
const index = csv(
|
||||
fs.readFileSync('docs/investmoz/v2/indice-documento-mestre.csv', 'utf8'),
|
||||
);
|
||||
const paragraphs = read('master-paragraphs.json');
|
||||
const headings = new Map();
|
||||
for (const row of index.slice(1)) {
|
||||
const title = row[5];
|
||||
if (!title) continue;
|
||||
const start = Number(row[2]),
|
||||
end = Number(row[3]);
|
||||
const body = paragraphs
|
||||
.filter((p) => p.p >= start && p.p <= end)
|
||||
.map((p) => p.text.replace(/\s+/g, ' ').trim())
|
||||
.join('\n');
|
||||
const key = title.trim().toLowerCase();
|
||||
headings.set(key, [
|
||||
...(headings.get(key) ?? []),
|
||||
{
|
||||
title,
|
||||
start,
|
||||
end,
|
||||
paragraphs: end - start + 1,
|
||||
contentSha256: crypto
|
||||
.createHash('sha256')
|
||||
.update(body)
|
||||
.digest('hex'),
|
||||
},
|
||||
]);
|
||||
}
|
||||
const repeated = [...headings.values()]
|
||||
.filter((g) => g.length > 1)
|
||||
.map((g) => ({
|
||||
title: g[0].title,
|
||||
occurrences: g,
|
||||
comparison:
|
||||
new Set(g.map((r) => r.contentSha256)).size > 1
|
||||
? 'historical_content_differs'
|
||||
: 'historical_content_identical',
|
||||
decision: 'pending_human_reconciliation_no_requirement_discarded',
|
||||
}));
|
||||
fs.writeFileSync(
|
||||
'docs/investmoz/v2/repeticoes-reconciliacao.json',
|
||||
JSON.stringify(
|
||||
{
|
||||
sourceSha256: index[1][1],
|
||||
basis: 'historical_saved_extraction_not_current_locked_original',
|
||||
repeated,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
records: records.length,
|
||||
duplicateGroups: [...groups.values()].filter((g) => g.length > 1)
|
||||
.length,
|
||||
repeatedGroups: repeated.length,
|
||||
differentGroups: repeated.filter(
|
||||
(g) => g.comparison === 'historical_content_differs',
|
||||
).length,
|
||||
}),
|
||||
);
|
||||
Reference in New Issue
Block a user