616 lines
16 KiB
TypeScript
616 lines
16 KiB
TypeScript
import type {
|
|
ProtectedTranslationTokens,
|
|
TranslationPayloadHealth,
|
|
TranslationCacheEntry,
|
|
TranslationLanguage,
|
|
} from "./types";
|
|
|
|
const CACHE_PREFIX = "tempo_translation_v1";
|
|
const CACHE_VERSION = "v1";
|
|
const CACHE_TTL = 7 * 24 * 60 * 60 * 1000;
|
|
const CACHE_MAX_ENTRIES = 200;
|
|
const CACHE_CLEANUP_INTERVAL = 12 * 60 * 60 * 1000;
|
|
const CLEANUP_MARKER_KEY = `${CACHE_PREFIX}:cleanup`;
|
|
const memoryCache = new Map<string, TranslationCacheEntry>();
|
|
|
|
const LANGUAGE_ALIASES: Record<string, string> = {
|
|
ar: "ar",
|
|
"ar-sa": "ar",
|
|
en: "en",
|
|
"en-gb": "en",
|
|
"en-us": "en",
|
|
es: "es",
|
|
"es-es": "es",
|
|
fr: "fr",
|
|
"fr-fr": "fr",
|
|
hi: "hi",
|
|
"hi-in": "hi",
|
|
pt: "pt",
|
|
"pt-mz": "pt",
|
|
"pt-pt": "pt",
|
|
zh: "zh-Hans",
|
|
"zh-cn": "zh-Hans",
|
|
"zh-hans": "zh-Hans",
|
|
};
|
|
|
|
const apiUnsafeNodes = "script, iframe, object, embed, link, meta";
|
|
const excludedTranslationNodes =
|
|
"code, pre, script, style, textarea, noscript, .no-translate, [data-no-translate], [translate='no']";
|
|
const INTERNAL_PLACEHOLDER_PREFIX = "¤TRX";
|
|
const BATCH_MAX_ITEMS = 8;
|
|
const BATCH_MAX_CHARS = 2200;
|
|
const TRANSLATION_PLACEHOLDER_PATTERN = /¤TRX[A-Z]+\d+¤/g;
|
|
const HTML_BATCH_ROOT_ATTR = "data-trx-batch-root";
|
|
const HTML_BATCH_NODE_ATTR = "data-trx-node";
|
|
|
|
export const fallbackTranslationLanguages: TranslationLanguage[] = [
|
|
{ code: "pt", name: "Português" },
|
|
{ code: "en", name: "English" },
|
|
{ code: "fr", name: "French" },
|
|
{ code: "es", name: "Spanish" },
|
|
{ code: "ar", name: "Arabic" },
|
|
{ code: "hi", name: "Hindi" },
|
|
{ code: "zh-Hans", name: "Chinese" },
|
|
];
|
|
|
|
export const DEFAULT_PROTECTED_TERMS = [
|
|
"Revista TEMPO",
|
|
"Grupo TEMPO",
|
|
"Tempo TV",
|
|
"Tempo Connect",
|
|
"Informação Pública",
|
|
"InvesteMoz",
|
|
"TEMPO",
|
|
] as const;
|
|
|
|
function getStorage(): Storage | null {
|
|
if (typeof window === "undefined") return null;
|
|
return window.localStorage;
|
|
}
|
|
|
|
function normalizeAlias(code: string): string {
|
|
return LANGUAGE_ALIASES[code.trim().toLowerCase()] ?? code.trim();
|
|
}
|
|
|
|
export function normalizeLanguageCode(
|
|
code: string,
|
|
availableLanguages: TranslationLanguage[],
|
|
): string | null {
|
|
if (!code) return null;
|
|
|
|
const normalized = normalizeAlias(code);
|
|
const supportedCodes = new Set(
|
|
availableLanguages.map((language) => normalizeAlias(language.code)),
|
|
);
|
|
|
|
return supportedCodes.has(normalized) ? normalized : null;
|
|
}
|
|
|
|
export function getBrowserLanguageCode(code: string): string {
|
|
switch (code) {
|
|
case "zh-Hans":
|
|
return "zh-CN";
|
|
default:
|
|
return code;
|
|
}
|
|
}
|
|
|
|
export function getLocaleForLanguage(code: string): string {
|
|
switch (normalizeAlias(code)) {
|
|
case "pt":
|
|
return "pt-MZ";
|
|
case "zh-Hans":
|
|
return "zh-CN";
|
|
default:
|
|
return normalizeAlias(code);
|
|
}
|
|
}
|
|
|
|
export function isRtlLanguage(code: string): boolean {
|
|
return normalizeAlias(code) === "ar";
|
|
}
|
|
|
|
export function readLanguageList(payload: unknown): TranslationLanguage[] {
|
|
const source = Array.isArray(payload)
|
|
? payload
|
|
: Array.isArray((payload as { value?: unknown[] })?.value)
|
|
? (payload as { value: unknown[] }).value
|
|
: Array.isArray((payload as { data?: unknown[] })?.data)
|
|
? (payload as { data: unknown[] }).data
|
|
: [];
|
|
|
|
return source
|
|
.filter((item): item is TranslationLanguage => {
|
|
return (
|
|
Boolean(item) &&
|
|
typeof item === "object" &&
|
|
typeof item.code === "string" &&
|
|
typeof item.name === "string"
|
|
);
|
|
})
|
|
.map((language) => ({
|
|
code: normalizeAlias(language.code),
|
|
name: language.name,
|
|
targets: Array.isArray(language.targets)
|
|
? language.targets.map((target) => normalizeAlias(target))
|
|
: undefined,
|
|
}));
|
|
}
|
|
|
|
export function withPortugueseFirst(
|
|
languages: TranslationLanguage[],
|
|
): TranslationLanguage[] {
|
|
const unique = new Map<string, TranslationLanguage>();
|
|
|
|
for (const language of languages) {
|
|
const code = normalizeAlias(language.code);
|
|
if (!unique.has(code)) {
|
|
unique.set(code, {
|
|
...language,
|
|
code,
|
|
name: code === "pt" ? "Português" : language.name,
|
|
});
|
|
}
|
|
}
|
|
|
|
const items = Array.from(unique.values()).sort((a, b) =>
|
|
a.name.localeCompare(b.name),
|
|
);
|
|
|
|
const portuguese =
|
|
items.find((language) => language.code === "pt") ?? fallbackTranslationLanguages[0];
|
|
|
|
return [portuguese, ...items.filter((language) => language.code !== "pt")];
|
|
}
|
|
|
|
export function hashTranslationText(value: string): string {
|
|
let hash = 2166136261;
|
|
|
|
for (let index = 0; index < value.length; index += 1) {
|
|
hash ^= value.charCodeAt(index);
|
|
hash = Math.imul(hash, 16777619);
|
|
}
|
|
|
|
return (hash >>> 0).toString(16);
|
|
}
|
|
|
|
export function createTranslationCacheKey(params: {
|
|
sourceLanguage: string;
|
|
targetLanguage: string;
|
|
isHtml: boolean;
|
|
text: string;
|
|
protectedTerms?: string[];
|
|
}): string {
|
|
return [
|
|
CACHE_PREFIX,
|
|
CACHE_VERSION,
|
|
params.sourceLanguage,
|
|
params.targetLanguage,
|
|
params.isHtml ? "html" : "text",
|
|
hashTranslationText(params.text),
|
|
hashTranslationText(
|
|
JSON.stringify(
|
|
collectProtectedTerms(params.protectedTerms).map((term) => term.toLowerCase()),
|
|
),
|
|
),
|
|
].join(":");
|
|
}
|
|
|
|
export function getMemoryCacheEntry(key: string): TranslationCacheEntry | null {
|
|
const entry = memoryCache.get(key);
|
|
if (!entry) return null;
|
|
|
|
if (entry.expiresAt <= Date.now()) {
|
|
memoryCache.delete(key);
|
|
return null;
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
export function setMemoryCacheEntry(key: string, value: string): TranslationCacheEntry {
|
|
const entry: TranslationCacheEntry = {
|
|
value,
|
|
createdAt: Date.now(),
|
|
expiresAt: Date.now() + CACHE_TTL,
|
|
};
|
|
|
|
memoryCache.set(key, entry);
|
|
return entry;
|
|
}
|
|
|
|
export function getPersistentCacheEntry(key: string): TranslationCacheEntry | null {
|
|
const storage = getStorage();
|
|
if (!storage) return null;
|
|
|
|
const raw = storage.getItem(key);
|
|
if (!raw) return null;
|
|
|
|
try {
|
|
const entry = JSON.parse(raw) as TranslationCacheEntry;
|
|
if (entry.expiresAt <= Date.now()) {
|
|
storage.removeItem(key);
|
|
return null;
|
|
}
|
|
|
|
return entry;
|
|
} catch {
|
|
storage.removeItem(key);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function setPersistentCacheEntry(key: string, entry: TranslationCacheEntry): void {
|
|
const storage = getStorage();
|
|
if (!storage) return;
|
|
|
|
storage.setItem(key, JSON.stringify(entry));
|
|
}
|
|
|
|
export function cleanupTranslationCache(): void {
|
|
const storage = getStorage();
|
|
if (!storage) return;
|
|
|
|
const lastCleanup = Number(storage.getItem(CLEANUP_MARKER_KEY) ?? "0");
|
|
if (lastCleanup && Date.now() - lastCleanup < CACHE_CLEANUP_INTERVAL) {
|
|
return;
|
|
}
|
|
|
|
const translationKeys: string[] = [];
|
|
|
|
for (let index = 0; index < storage.length; index += 1) {
|
|
const key = storage.key(index);
|
|
if (key && key.startsWith(`${CACHE_PREFIX}:${CACHE_VERSION}:`)) {
|
|
translationKeys.push(key);
|
|
}
|
|
}
|
|
|
|
const validEntries: Array<{ key: string; entry: TranslationCacheEntry }> = [];
|
|
|
|
for (const key of translationKeys) {
|
|
const entry = getPersistentCacheEntry(key);
|
|
if (entry) {
|
|
validEntries.push({ key, entry });
|
|
}
|
|
}
|
|
|
|
validEntries
|
|
.sort((a, b) => a.entry.createdAt - b.entry.createdAt)
|
|
.slice(0, Math.max(0, validEntries.length - CACHE_MAX_ENTRIES))
|
|
.forEach(({ key }) => storage.removeItem(key));
|
|
|
|
storage.setItem(CLEANUP_MARKER_KEY, String(Date.now()));
|
|
}
|
|
|
|
function createPlaceholder(prefix: string, index: number): string {
|
|
return `${INTERNAL_PLACEHOLDER_PREFIX}${prefix}${index}¤`;
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
export function collectProtectedTerms(terms?: Array<string | null | undefined>): string[] {
|
|
const combined = [...DEFAULT_PROTECTED_TERMS, ...(terms ?? [])]
|
|
.map((term) => String(term ?? "").trim())
|
|
.filter(Boolean);
|
|
|
|
return Array.from(new Set(combined)).sort((a, b) => b.length - a.length);
|
|
}
|
|
|
|
function protectMatches(
|
|
input: string,
|
|
pattern: RegExp,
|
|
replacements: Map<string, string>,
|
|
prefix: string,
|
|
): string {
|
|
return input.replace(pattern, (match) => {
|
|
const placeholder = createPlaceholder(prefix, replacements.size);
|
|
replacements.set(placeholder, match);
|
|
return placeholder;
|
|
});
|
|
}
|
|
|
|
function protectExcludedHtmlBlocks(
|
|
input: string,
|
|
replacements: Map<string, string>,
|
|
): string {
|
|
if (typeof window === "undefined") return input;
|
|
|
|
const documentFragment = new DOMParser().parseFromString(input, "text/html");
|
|
const nodes = Array.from(documentFragment.body.querySelectorAll(excludedTranslationNodes));
|
|
|
|
nodes.forEach((node) => {
|
|
const placeholder = createPlaceholder("HTML", replacements.size);
|
|
replacements.set(placeholder, node.outerHTML);
|
|
node.replaceWith(documentFragment.createTextNode(placeholder));
|
|
});
|
|
|
|
return documentFragment.body.innerHTML;
|
|
}
|
|
|
|
export function protectTranslationTokens(
|
|
input: string,
|
|
options?: { isHtml?: boolean; protectedTerms?: string[] },
|
|
): ProtectedTranslationTokens {
|
|
const replacements = new Map<string, string>();
|
|
let output = String(input ?? "");
|
|
|
|
if (options?.isHtml) {
|
|
output = protectExcludedHtmlBlocks(output, replacements);
|
|
}
|
|
|
|
output = protectMatches(
|
|
output,
|
|
/https?:\/\/[^\s"'<>]+/gi,
|
|
replacements,
|
|
"URL",
|
|
);
|
|
output = protectMatches(
|
|
output,
|
|
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
|
|
replacements,
|
|
"EMAIL",
|
|
);
|
|
output = protectMatches(
|
|
output,
|
|
/\{\{[^{}]+\}\}|\{[a-zA-Z0-9_.-]+\}|%(\d+\$)?[sd]|:[a-zA-Z_][a-zA-Z0-9_]*/g,
|
|
replacements,
|
|
"TOKEN",
|
|
);
|
|
|
|
for (const term of collectProtectedTerms(options?.protectedTerms)) {
|
|
const escapedTerm = escapeRegExp(term);
|
|
const pattern = new RegExp(`(^|[^\\p{L}\\p{N}])(${escapedTerm})(?=$|[^\\p{L}\\p{N}])`, "giu");
|
|
|
|
output = output.replace(pattern, (match, prefix, protectedValue) => {
|
|
const placeholder = createPlaceholder("TERM", replacements.size);
|
|
replacements.set(placeholder, protectedValue);
|
|
return `${prefix}${placeholder}`;
|
|
});
|
|
}
|
|
|
|
return { text: output, replacements };
|
|
}
|
|
|
|
export function restoreTranslationTokens(
|
|
input: string,
|
|
replacements: Map<string, string>,
|
|
): string {
|
|
let output = String(input ?? "");
|
|
|
|
for (const [placeholder, original] of replacements.entries()) {
|
|
output = output.split(placeholder).join(original);
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
export function getTranslationPayloadHealth(
|
|
input: string,
|
|
options?: { isHtml?: boolean; protectedTerms?: string[] },
|
|
): TranslationPayloadHealth {
|
|
const source = String(input ?? "");
|
|
if (!source.trim()) {
|
|
return {
|
|
shouldTranslate: false,
|
|
normalizedText: source,
|
|
};
|
|
}
|
|
|
|
if (options?.isHtml && typeof window !== "undefined") {
|
|
const documentFragment = new DOMParser().parseFromString(source, "text/html");
|
|
const walker = document.createTreeWalker(
|
|
documentFragment.body,
|
|
NodeFilter.SHOW_TEXT,
|
|
{
|
|
acceptNode(node) {
|
|
return isSkippableTextNode(node as Text)
|
|
? NodeFilter.FILTER_REJECT
|
|
: NodeFilter.FILTER_ACCEPT;
|
|
},
|
|
},
|
|
);
|
|
|
|
if (!walker.nextNode()) {
|
|
return {
|
|
shouldTranslate: false,
|
|
normalizedText: source,
|
|
};
|
|
}
|
|
}
|
|
|
|
const protectedPayload = protectTranslationTokens(source, options);
|
|
const remainder = protectedPayload.text
|
|
.replace(TRANSLATION_PLACEHOLDER_PATTERN, "")
|
|
.trim();
|
|
|
|
if (!remainder) {
|
|
return {
|
|
shouldTranslate: false,
|
|
normalizedText: source,
|
|
};
|
|
}
|
|
|
|
const hasLetters = /[\p{L}]/u.test(remainder);
|
|
if (!hasLetters) {
|
|
return {
|
|
shouldTranslate: false,
|
|
normalizedText: source,
|
|
};
|
|
}
|
|
|
|
return {
|
|
shouldTranslate: true,
|
|
normalizedText: source,
|
|
};
|
|
}
|
|
|
|
export function sanitizeTranslatedHtml(html: string): string {
|
|
if (typeof window === "undefined" || !html) return "";
|
|
|
|
const documentFragment = new DOMParser().parseFromString(String(html), "text/html");
|
|
documentFragment.querySelectorAll(apiUnsafeNodes).forEach((node) => node.remove());
|
|
|
|
documentFragment.querySelectorAll("*").forEach((node) => {
|
|
for (const attribute of Array.from(node.attributes)) {
|
|
const name = attribute.name.toLowerCase();
|
|
const value = attribute.value.trim().toLowerCase();
|
|
|
|
if (
|
|
name.startsWith("on") ||
|
|
value.startsWith("javascript:") ||
|
|
value.startsWith("data:text/html") ||
|
|
value.startsWith("vbscript:")
|
|
) {
|
|
node.removeAttribute(attribute.name);
|
|
}
|
|
}
|
|
});
|
|
|
|
return documentFragment.body.innerHTML;
|
|
}
|
|
|
|
function isSkippableTextNode(node: Text): boolean {
|
|
const parent = node.parentElement;
|
|
const value = node.textContent ?? "";
|
|
|
|
if (!value.trim()) return true;
|
|
if (!parent) return true;
|
|
if (parent.closest(excludedTranslationNodes)) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
function createBatchPayload(values: string[]): {
|
|
payload: string;
|
|
} {
|
|
if (typeof window === "undefined") {
|
|
return {
|
|
payload: values.join(" "),
|
|
};
|
|
}
|
|
|
|
const documentFragment = document.implementation.createHTMLDocument("");
|
|
const root = documentFragment.createElement("div");
|
|
root.setAttribute(HTML_BATCH_ROOT_ATTR, "1");
|
|
|
|
values.forEach((value, index) => {
|
|
const node = documentFragment.createElement("span");
|
|
node.setAttribute(HTML_BATCH_NODE_ATTR, String(index));
|
|
node.textContent = value;
|
|
root.appendChild(node);
|
|
});
|
|
|
|
documentFragment.body.appendChild(root);
|
|
|
|
return { payload: root.outerHTML };
|
|
}
|
|
|
|
function parseBatchResponse(
|
|
translated: string,
|
|
originalValues: string[],
|
|
): string[] {
|
|
if (typeof window === "undefined") {
|
|
return originalValues;
|
|
}
|
|
|
|
const documentFragment = new DOMParser().parseFromString(translated, "text/html");
|
|
const root = documentFragment.body.querySelector(`[${HTML_BATCH_ROOT_ATTR}]`);
|
|
if (!root) {
|
|
return originalValues;
|
|
}
|
|
|
|
const nodes = Array.from(root.querySelectorAll(`[${HTML_BATCH_NODE_ATTR}]`));
|
|
if (nodes.length !== originalValues.length) {
|
|
return originalValues;
|
|
}
|
|
|
|
return originalValues.map((originalValue, index) => {
|
|
const node = nodes.find(
|
|
(candidate) => candidate.getAttribute(HTML_BATCH_NODE_ATTR) === String(index),
|
|
);
|
|
|
|
return node?.textContent?.trim() ? node.textContent : originalValue;
|
|
});
|
|
}
|
|
|
|
function buildTextBatches(nodes: Text[]): Text[][] {
|
|
const batches: Text[][] = [];
|
|
let currentBatch: Text[] = [];
|
|
let currentLength = 0;
|
|
|
|
for (const node of nodes) {
|
|
const textLength = (node.textContent ?? "").length;
|
|
const shouldFlush =
|
|
currentBatch.length >= BATCH_MAX_ITEMS ||
|
|
currentLength + textLength > BATCH_MAX_CHARS;
|
|
|
|
if (shouldFlush && currentBatch.length > 0) {
|
|
batches.push(currentBatch);
|
|
currentBatch = [];
|
|
currentLength = 0;
|
|
}
|
|
|
|
currentBatch.push(node);
|
|
currentLength += textLength;
|
|
}
|
|
|
|
if (currentBatch.length > 0) {
|
|
batches.push(currentBatch);
|
|
}
|
|
|
|
return batches;
|
|
}
|
|
|
|
export async function translateHtmlByTextNodes(params: {
|
|
html: string;
|
|
protectedTerms?: string[];
|
|
translateText: (text: string, options?: { protectedTerms?: string[]; isHtml?: boolean }) => Promise<string>;
|
|
}): Promise<string> {
|
|
if (typeof window === "undefined" || !params.html.trim()) {
|
|
return params.html;
|
|
}
|
|
|
|
const documentFragment = new DOMParser().parseFromString(params.html, "text/html");
|
|
const walker = document.createTreeWalker(
|
|
documentFragment.body,
|
|
NodeFilter.SHOW_TEXT,
|
|
{
|
|
acceptNode(node) {
|
|
return isSkippableTextNode(node as Text)
|
|
? NodeFilter.FILTER_REJECT
|
|
: NodeFilter.FILTER_ACCEPT;
|
|
},
|
|
},
|
|
);
|
|
|
|
const textNodes: Text[] = [];
|
|
let currentNode = walker.nextNode();
|
|
while (currentNode) {
|
|
textNodes.push(currentNode as Text);
|
|
currentNode = walker.nextNode();
|
|
}
|
|
|
|
if (textNodes.length === 0) {
|
|
return sanitizeTranslatedHtml(documentFragment.body.innerHTML);
|
|
}
|
|
|
|
for (const batch of buildTextBatches(textNodes)) {
|
|
const originalValues = batch.map((node) => node.textContent ?? "");
|
|
const { payload } = createBatchPayload(originalValues);
|
|
const translatedPayload = await params.translateText(payload, {
|
|
protectedTerms: params.protectedTerms,
|
|
isHtml: true,
|
|
});
|
|
|
|
const translatedValues = parseBatchResponse(translatedPayload, originalValues);
|
|
|
|
batch.forEach((node, index) => {
|
|
node.textContent = translatedValues[index];
|
|
});
|
|
}
|
|
|
|
return sanitizeTranslatedHtml(documentFragment.body.innerHTML);
|
|
}
|