92 lines
3.0 KiB
JavaScript
92 lines
3.0 KiB
JavaScript
const fs = require('fs');
|
|
const d = 'storage/app/private/investmoz-v2/';
|
|
const read = (f) =>
|
|
JSON.parse(fs.readFileSync(d + f, 'utf8').replace(/^\uFEFF/, ''));
|
|
const norm = (s) =>
|
|
s
|
|
.replace(/[\u200b\ufeff]/g, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
.replace(
|
|
'2. Visão, objectivos e princípios',
|
|
'2. Visão, objetivos e princípios',
|
|
);
|
|
const clean = (a) =>
|
|
a
|
|
.filter((p) => !/(ndice|toc)/i.test(p.style) && norm(p.text))
|
|
.map((p) => ({ ...p, text: norm(p.text) }));
|
|
const a = clean(read('master-paragraphs.json')),
|
|
b = clean(read('master-paragraphs-reopened.json'));
|
|
const pairs = [];
|
|
function row(x0, x1, y0, y1, reverse) {
|
|
const size = y1 - y0;
|
|
let prev = new Uint32Array(size + 1),
|
|
curr = new Uint32Array(size + 1);
|
|
for (let i = 0; i < x1 - x0; i++) {
|
|
curr[0] = 0;
|
|
const text = a[reverse ? x1 - 1 - i : x0 + i].text;
|
|
for (let j = 1; j <= size; j++)
|
|
curr[j] =
|
|
text === b[reverse ? y1 - j : y0 + j - 1].text
|
|
? prev[j - 1] + 1
|
|
: Math.max(prev[j], curr[j - 1]);
|
|
[prev, curr] = [curr, prev];
|
|
}
|
|
return prev;
|
|
}
|
|
function align(x0, x1, y0, y1) {
|
|
while (x0 < x1 && y0 < y1 && a[x0].text === b[y0].text)
|
|
pairs.push([x0++, y0++]);
|
|
const tail = [];
|
|
while (x0 < x1 && y0 < y1 && a[x1 - 1].text === b[y1 - 1].text)
|
|
tail.push([--x1, --y1]);
|
|
if (x0 < x1 && y0 < y1) {
|
|
if (x1 - x0 === 1) {
|
|
for (let j = y0; j < y1; j++)
|
|
if (a[x0].text === b[j].text) {
|
|
pairs.push([x0, j]);
|
|
break;
|
|
}
|
|
} else {
|
|
let mid = (x0 + x1) >> 1;
|
|
let f = row(x0, mid, y0, y1, false),
|
|
r = row(mid, x1, y0, y1, true),
|
|
split = 0,
|
|
best = -1;
|
|
for (let j = 0; j <= y1 - y0; j++)
|
|
if (f[j] + r[y1 - y0 - j] > best) {
|
|
best = f[j] + r[y1 - y0 - j];
|
|
split = j;
|
|
}
|
|
align(x0, mid, y0, y0 + split);
|
|
align(mid, x1, y0 + split, y1);
|
|
}
|
|
}
|
|
pairs.push(...tail.reverse());
|
|
}
|
|
align(0, a.length, 0, b.length);
|
|
const ma = new Set(pairs.map((p) => p[0])),
|
|
mb = new Set(pairs.map((p) => p[1]));
|
|
const removed = a.filter((p, i) => !ma.has(i)),
|
|
added = b.filter((p, i) => !mb.has(i));
|
|
const result = {
|
|
method: 'LCS paragraph alignment; exclude TOC and empty paragraphs; normalize heading orthography only',
|
|
matched: pairs.length,
|
|
removed,
|
|
added,
|
|
mapping: pairs.map(([i, j]) => ({ old: a[i].p, current: b[j].p })),
|
|
};
|
|
fs.writeFileSync(
|
|
d + 'master-aligned-changes.json',
|
|
JSON.stringify(result, null, 2),
|
|
);
|
|
console.log(
|
|
JSON.stringify({
|
|
matched: pairs.length,
|
|
removed: removed.length,
|
|
added: added.length,
|
|
addedTexts: added.slice(0, 8),
|
|
removedFirst: removed.slice(0, 4),
|
|
}),
|
|
);
|