.
This commit is contained in:
@@ -0,0 +1,615 @@
|
||||
import type {
|
||||
ProtectedTranslationTokens,
|
||||
TranslationPayloadHealth,
|
||||
TranslationCacheEntry,
|
||||
TranslationLanguage,
|
||||
} from "./types";
|
||||
|
||||
const CACHE_PREFIX = "tempo_translation_v1";
|
||||
const CACHE_VERSION = "v1";
|
||||
const CACHE_TTL = 7 * 24 * 60 * 60 * 1000;
|
||||
const CACHE_MAX_ENTRIES = 200;
|
||||
const CACHE_CLEANUP_INTERVAL = 12 * 60 * 60 * 1000;
|
||||
const CLEANUP_MARKER_KEY = `${CACHE_PREFIX}:cleanup`;
|
||||
const memoryCache = new Map<string, TranslationCacheEntry>();
|
||||
|
||||
const LANGUAGE_ALIASES: Record<string, string> = {
|
||||
ar: "ar",
|
||||
"ar-sa": "ar",
|
||||
en: "en",
|
||||
"en-gb": "en",
|
||||
"en-us": "en",
|
||||
es: "es",
|
||||
"es-es": "es",
|
||||
fr: "fr",
|
||||
"fr-fr": "fr",
|
||||
hi: "hi",
|
||||
"hi-in": "hi",
|
||||
pt: "pt",
|
||||
"pt-mz": "pt",
|
||||
"pt-pt": "pt",
|
||||
zh: "zh-Hans",
|
||||
"zh-cn": "zh-Hans",
|
||||
"zh-hans": "zh-Hans",
|
||||
};
|
||||
|
||||
const apiUnsafeNodes = "script, iframe, object, embed, link, meta";
|
||||
const excludedTranslationNodes =
|
||||
"code, pre, script, style, textarea, noscript, .no-translate, [data-no-translate], [translate='no']";
|
||||
const INTERNAL_PLACEHOLDER_PREFIX = "¤TRX";
|
||||
const BATCH_MAX_ITEMS = 8;
|
||||
const BATCH_MAX_CHARS = 2200;
|
||||
const TRANSLATION_PLACEHOLDER_PATTERN = /¤TRX[A-Z]+\d+¤/g;
|
||||
const HTML_BATCH_ROOT_ATTR = "data-trx-batch-root";
|
||||
const HTML_BATCH_NODE_ATTR = "data-trx-node";
|
||||
|
||||
export const fallbackTranslationLanguages: TranslationLanguage[] = [
|
||||
{ code: "pt", name: "Português" },
|
||||
{ code: "en", name: "English" },
|
||||
{ code: "fr", name: "French" },
|
||||
{ code: "es", name: "Spanish" },
|
||||
{ code: "ar", name: "Arabic" },
|
||||
{ code: "hi", name: "Hindi" },
|
||||
{ code: "zh-Hans", name: "Chinese" },
|
||||
];
|
||||
|
||||
export const DEFAULT_PROTECTED_TERMS = [
|
||||
"Revista TEMPO",
|
||||
"Grupo TEMPO",
|
||||
"Tempo TV",
|
||||
"Tempo Connect",
|
||||
"Informação Pública",
|
||||
"InvesteMoz",
|
||||
"TEMPO",
|
||||
] as const;
|
||||
|
||||
function getStorage(): Storage | null {
|
||||
if (typeof window === "undefined") return null;
|
||||
return window.localStorage;
|
||||
}
|
||||
|
||||
function normalizeAlias(code: string): string {
|
||||
return LANGUAGE_ALIASES[code.trim().toLowerCase()] ?? code.trim();
|
||||
}
|
||||
|
||||
export function normalizeLanguageCode(
|
||||
code: string,
|
||||
availableLanguages: TranslationLanguage[],
|
||||
): string | null {
|
||||
if (!code) return null;
|
||||
|
||||
const normalized = normalizeAlias(code);
|
||||
const supportedCodes = new Set(
|
||||
availableLanguages.map((language) => normalizeAlias(language.code)),
|
||||
);
|
||||
|
||||
return supportedCodes.has(normalized) ? normalized : null;
|
||||
}
|
||||
|
||||
export function getBrowserLanguageCode(code: string): string {
|
||||
switch (code) {
|
||||
case "zh-Hans":
|
||||
return "zh-CN";
|
||||
default:
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
export function getLocaleForLanguage(code: string): string {
|
||||
switch (normalizeAlias(code)) {
|
||||
case "pt":
|
||||
return "pt-MZ";
|
||||
case "zh-Hans":
|
||||
return "zh-CN";
|
||||
default:
|
||||
return normalizeAlias(code);
|
||||
}
|
||||
}
|
||||
|
||||
export function isRtlLanguage(code: string): boolean {
|
||||
return normalizeAlias(code) === "ar";
|
||||
}
|
||||
|
||||
export function readLanguageList(payload: unknown): TranslationLanguage[] {
|
||||
const source = Array.isArray(payload)
|
||||
? payload
|
||||
: Array.isArray((payload as { value?: unknown[] })?.value)
|
||||
? (payload as { value: unknown[] }).value
|
||||
: Array.isArray((payload as { data?: unknown[] })?.data)
|
||||
? (payload as { data: unknown[] }).data
|
||||
: [];
|
||||
|
||||
return source
|
||||
.filter((item): item is TranslationLanguage => {
|
||||
return (
|
||||
Boolean(item) &&
|
||||
typeof item === "object" &&
|
||||
typeof item.code === "string" &&
|
||||
typeof item.name === "string"
|
||||
);
|
||||
})
|
||||
.map((language) => ({
|
||||
code: normalizeAlias(language.code),
|
||||
name: language.name,
|
||||
targets: Array.isArray(language.targets)
|
||||
? language.targets.map((target) => normalizeAlias(target))
|
||||
: undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
export function withPortugueseFirst(
|
||||
languages: TranslationLanguage[],
|
||||
): TranslationLanguage[] {
|
||||
const unique = new Map<string, TranslationLanguage>();
|
||||
|
||||
for (const language of languages) {
|
||||
const code = normalizeAlias(language.code);
|
||||
if (!unique.has(code)) {
|
||||
unique.set(code, {
|
||||
...language,
|
||||
code,
|
||||
name: code === "pt" ? "Português" : language.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const items = Array.from(unique.values()).sort((a, b) =>
|
||||
a.name.localeCompare(b.name),
|
||||
);
|
||||
|
||||
const portuguese =
|
||||
items.find((language) => language.code === "pt") ?? fallbackTranslationLanguages[0];
|
||||
|
||||
return [portuguese, ...items.filter((language) => language.code !== "pt")];
|
||||
}
|
||||
|
||||
export function hashTranslationText(value: string): string {
|
||||
let hash = 2166136261;
|
||||
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash ^= value.charCodeAt(index);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
|
||||
return (hash >>> 0).toString(16);
|
||||
}
|
||||
|
||||
export function createTranslationCacheKey(params: {
|
||||
sourceLanguage: string;
|
||||
targetLanguage: string;
|
||||
isHtml: boolean;
|
||||
text: string;
|
||||
protectedTerms?: string[];
|
||||
}): string {
|
||||
return [
|
||||
CACHE_PREFIX,
|
||||
CACHE_VERSION,
|
||||
params.sourceLanguage,
|
||||
params.targetLanguage,
|
||||
params.isHtml ? "html" : "text",
|
||||
hashTranslationText(params.text),
|
||||
hashTranslationText(
|
||||
JSON.stringify(
|
||||
collectProtectedTerms(params.protectedTerms).map((term) => term.toLowerCase()),
|
||||
),
|
||||
),
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export function getMemoryCacheEntry(key: string): TranslationCacheEntry | null {
|
||||
const entry = memoryCache.get(key);
|
||||
if (!entry) return null;
|
||||
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
memoryCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setMemoryCacheEntry(key: string, value: string): TranslationCacheEntry {
|
||||
const entry: TranslationCacheEntry = {
|
||||
value,
|
||||
createdAt: Date.now(),
|
||||
expiresAt: Date.now() + CACHE_TTL,
|
||||
};
|
||||
|
||||
memoryCache.set(key, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function getPersistentCacheEntry(key: string): TranslationCacheEntry | null {
|
||||
const storage = getStorage();
|
||||
if (!storage) return null;
|
||||
|
||||
const raw = storage.getItem(key);
|
||||
if (!raw) return null;
|
||||
|
||||
try {
|
||||
const entry = JSON.parse(raw) as TranslationCacheEntry;
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
storage.removeItem(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry;
|
||||
} catch {
|
||||
storage.removeItem(key);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function setPersistentCacheEntry(key: string, entry: TranslationCacheEntry): void {
|
||||
const storage = getStorage();
|
||||
if (!storage) return;
|
||||
|
||||
storage.setItem(key, JSON.stringify(entry));
|
||||
}
|
||||
|
||||
export function cleanupTranslationCache(): void {
|
||||
const storage = getStorage();
|
||||
if (!storage) return;
|
||||
|
||||
const lastCleanup = Number(storage.getItem(CLEANUP_MARKER_KEY) ?? "0");
|
||||
if (lastCleanup && Date.now() - lastCleanup < CACHE_CLEANUP_INTERVAL) {
|
||||
return;
|
||||
}
|
||||
|
||||
const translationKeys: string[] = [];
|
||||
|
||||
for (let index = 0; index < storage.length; index += 1) {
|
||||
const key = storage.key(index);
|
||||
if (key && key.startsWith(`${CACHE_PREFIX}:${CACHE_VERSION}:`)) {
|
||||
translationKeys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const validEntries: Array<{ key: string; entry: TranslationCacheEntry }> = [];
|
||||
|
||||
for (const key of translationKeys) {
|
||||
const entry = getPersistentCacheEntry(key);
|
||||
if (entry) {
|
||||
validEntries.push({ key, entry });
|
||||
}
|
||||
}
|
||||
|
||||
validEntries
|
||||
.sort((a, b) => a.entry.createdAt - b.entry.createdAt)
|
||||
.slice(0, Math.max(0, validEntries.length - CACHE_MAX_ENTRIES))
|
||||
.forEach(({ key }) => storage.removeItem(key));
|
||||
|
||||
storage.setItem(CLEANUP_MARKER_KEY, String(Date.now()));
|
||||
}
|
||||
|
||||
function createPlaceholder(prefix: string, index: number): string {
|
||||
return `${INTERNAL_PLACEHOLDER_PREFIX}${prefix}${index}¤`;
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
export function collectProtectedTerms(terms?: Array<string | null | undefined>): string[] {
|
||||
const combined = [...DEFAULT_PROTECTED_TERMS, ...(terms ?? [])]
|
||||
.map((term) => String(term ?? "").trim())
|
||||
.filter(Boolean);
|
||||
|
||||
return Array.from(new Set(combined)).sort((a, b) => b.length - a.length);
|
||||
}
|
||||
|
||||
function protectMatches(
|
||||
input: string,
|
||||
pattern: RegExp,
|
||||
replacements: Map<string, string>,
|
||||
prefix: string,
|
||||
): string {
|
||||
return input.replace(pattern, (match) => {
|
||||
const placeholder = createPlaceholder(prefix, replacements.size);
|
||||
replacements.set(placeholder, match);
|
||||
return placeholder;
|
||||
});
|
||||
}
|
||||
|
||||
function protectExcludedHtmlBlocks(
|
||||
input: string,
|
||||
replacements: Map<string, string>,
|
||||
): string {
|
||||
if (typeof window === "undefined") return input;
|
||||
|
||||
const documentFragment = new DOMParser().parseFromString(input, "text/html");
|
||||
const nodes = Array.from(documentFragment.body.querySelectorAll(excludedTranslationNodes));
|
||||
|
||||
nodes.forEach((node) => {
|
||||
const placeholder = createPlaceholder("HTML", replacements.size);
|
||||
replacements.set(placeholder, node.outerHTML);
|
||||
node.replaceWith(documentFragment.createTextNode(placeholder));
|
||||
});
|
||||
|
||||
return documentFragment.body.innerHTML;
|
||||
}
|
||||
|
||||
export function protectTranslationTokens(
|
||||
input: string,
|
||||
options?: { isHtml?: boolean; protectedTerms?: string[] },
|
||||
): ProtectedTranslationTokens {
|
||||
const replacements = new Map<string, string>();
|
||||
let output = String(input ?? "");
|
||||
|
||||
if (options?.isHtml) {
|
||||
output = protectExcludedHtmlBlocks(output, replacements);
|
||||
}
|
||||
|
||||
output = protectMatches(
|
||||
output,
|
||||
/https?:\/\/[^\s"'<>]+/gi,
|
||||
replacements,
|
||||
"URL",
|
||||
);
|
||||
output = protectMatches(
|
||||
output,
|
||||
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
|
||||
replacements,
|
||||
"EMAIL",
|
||||
);
|
||||
output = protectMatches(
|
||||
output,
|
||||
/\{\{[^{}]+\}\}|\{[a-zA-Z0-9_.-]+\}|%(\d+\$)?[sd]|:[a-zA-Z_][a-zA-Z0-9_]*/g,
|
||||
replacements,
|
||||
"TOKEN",
|
||||
);
|
||||
|
||||
for (const term of collectProtectedTerms(options?.protectedTerms)) {
|
||||
const escapedTerm = escapeRegExp(term);
|
||||
const pattern = new RegExp(`(^|[^\\p{L}\\p{N}])(${escapedTerm})(?=$|[^\\p{L}\\p{N}])`, "giu");
|
||||
|
||||
output = output.replace(pattern, (match, prefix, protectedValue) => {
|
||||
const placeholder = createPlaceholder("TERM", replacements.size);
|
||||
replacements.set(placeholder, protectedValue);
|
||||
return `${prefix}${placeholder}`;
|
||||
});
|
||||
}
|
||||
|
||||
return { text: output, replacements };
|
||||
}
|
||||
|
||||
export function restoreTranslationTokens(
|
||||
input: string,
|
||||
replacements: Map<string, string>,
|
||||
): string {
|
||||
let output = String(input ?? "");
|
||||
|
||||
for (const [placeholder, original] of replacements.entries()) {
|
||||
output = output.split(placeholder).join(original);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
export function getTranslationPayloadHealth(
|
||||
input: string,
|
||||
options?: { isHtml?: boolean; protectedTerms?: string[] },
|
||||
): TranslationPayloadHealth {
|
||||
const source = String(input ?? "");
|
||||
if (!source.trim()) {
|
||||
return {
|
||||
shouldTranslate: false,
|
||||
normalizedText: source,
|
||||
};
|
||||
}
|
||||
|
||||
if (options?.isHtml && typeof window !== "undefined") {
|
||||
const documentFragment = new DOMParser().parseFromString(source, "text/html");
|
||||
const walker = document.createTreeWalker(
|
||||
documentFragment.body,
|
||||
NodeFilter.SHOW_TEXT,
|
||||
{
|
||||
acceptNode(node) {
|
||||
return isSkippableTextNode(node as Text)
|
||||
? NodeFilter.FILTER_REJECT
|
||||
: NodeFilter.FILTER_ACCEPT;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (!walker.nextNode()) {
|
||||
return {
|
||||
shouldTranslate: false,
|
||||
normalizedText: source,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const protectedPayload = protectTranslationTokens(source, options);
|
||||
const remainder = protectedPayload.text
|
||||
.replace(TRANSLATION_PLACEHOLDER_PATTERN, "")
|
||||
.trim();
|
||||
|
||||
if (!remainder) {
|
||||
return {
|
||||
shouldTranslate: false,
|
||||
normalizedText: source,
|
||||
};
|
||||
}
|
||||
|
||||
const hasLetters = /[\p{L}]/u.test(remainder);
|
||||
if (!hasLetters) {
|
||||
return {
|
||||
shouldTranslate: false,
|
||||
normalizedText: source,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
shouldTranslate: true,
|
||||
normalizedText: source,
|
||||
};
|
||||
}
|
||||
|
||||
export function sanitizeTranslatedHtml(html: string): string {
|
||||
if (typeof window === "undefined" || !html) return "";
|
||||
|
||||
const documentFragment = new DOMParser().parseFromString(String(html), "text/html");
|
||||
documentFragment.querySelectorAll(apiUnsafeNodes).forEach((node) => node.remove());
|
||||
|
||||
documentFragment.querySelectorAll("*").forEach((node) => {
|
||||
for (const attribute of Array.from(node.attributes)) {
|
||||
const name = attribute.name.toLowerCase();
|
||||
const value = attribute.value.trim().toLowerCase();
|
||||
|
||||
if (
|
||||
name.startsWith("on") ||
|
||||
value.startsWith("javascript:") ||
|
||||
value.startsWith("data:text/html") ||
|
||||
value.startsWith("vbscript:")
|
||||
) {
|
||||
node.removeAttribute(attribute.name);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return documentFragment.body.innerHTML;
|
||||
}
|
||||
|
||||
function isSkippableTextNode(node: Text): boolean {
|
||||
const parent = node.parentElement;
|
||||
const value = node.textContent ?? "";
|
||||
|
||||
if (!value.trim()) return true;
|
||||
if (!parent) return true;
|
||||
if (parent.closest(excludedTranslationNodes)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function createBatchPayload(values: string[]): {
|
||||
payload: string;
|
||||
} {
|
||||
if (typeof window === "undefined") {
|
||||
return {
|
||||
payload: values.join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
const documentFragment = document.implementation.createHTMLDocument("");
|
||||
const root = documentFragment.createElement("div");
|
||||
root.setAttribute(HTML_BATCH_ROOT_ATTR, "1");
|
||||
|
||||
values.forEach((value, index) => {
|
||||
const node = documentFragment.createElement("span");
|
||||
node.setAttribute(HTML_BATCH_NODE_ATTR, String(index));
|
||||
node.textContent = value;
|
||||
root.appendChild(node);
|
||||
});
|
||||
|
||||
documentFragment.body.appendChild(root);
|
||||
|
||||
return { payload: root.outerHTML };
|
||||
}
|
||||
|
||||
function parseBatchResponse(
|
||||
translated: string,
|
||||
originalValues: string[],
|
||||
): string[] {
|
||||
if (typeof window === "undefined") {
|
||||
return originalValues;
|
||||
}
|
||||
|
||||
const documentFragment = new DOMParser().parseFromString(translated, "text/html");
|
||||
const root = documentFragment.body.querySelector(`[${HTML_BATCH_ROOT_ATTR}]`);
|
||||
if (!root) {
|
||||
return originalValues;
|
||||
}
|
||||
|
||||
const nodes = Array.from(root.querySelectorAll(`[${HTML_BATCH_NODE_ATTR}]`));
|
||||
if (nodes.length !== originalValues.length) {
|
||||
return originalValues;
|
||||
}
|
||||
|
||||
return originalValues.map((originalValue, index) => {
|
||||
const node = nodes.find(
|
||||
(candidate) => candidate.getAttribute(HTML_BATCH_NODE_ATTR) === String(index),
|
||||
);
|
||||
|
||||
return node?.textContent?.trim() ? node.textContent : originalValue;
|
||||
});
|
||||
}
|
||||
|
||||
function buildTextBatches(nodes: Text[]): Text[][] {
|
||||
const batches: Text[][] = [];
|
||||
let currentBatch: Text[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
for (const node of nodes) {
|
||||
const textLength = (node.textContent ?? "").length;
|
||||
const shouldFlush =
|
||||
currentBatch.length >= BATCH_MAX_ITEMS ||
|
||||
currentLength + textLength > BATCH_MAX_CHARS;
|
||||
|
||||
if (shouldFlush && currentBatch.length > 0) {
|
||||
batches.push(currentBatch);
|
||||
currentBatch = [];
|
||||
currentLength = 0;
|
||||
}
|
||||
|
||||
currentBatch.push(node);
|
||||
currentLength += textLength;
|
||||
}
|
||||
|
||||
if (currentBatch.length > 0) {
|
||||
batches.push(currentBatch);
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
|
||||
export async function translateHtmlByTextNodes(params: {
|
||||
html: string;
|
||||
protectedTerms?: string[];
|
||||
translateText: (text: string, options?: { protectedTerms?: string[]; isHtml?: boolean }) => Promise<string>;
|
||||
}): Promise<string> {
|
||||
if (typeof window === "undefined" || !params.html.trim()) {
|
||||
return params.html;
|
||||
}
|
||||
|
||||
const documentFragment = new DOMParser().parseFromString(params.html, "text/html");
|
||||
const walker = document.createTreeWalker(
|
||||
documentFragment.body,
|
||||
NodeFilter.SHOW_TEXT,
|
||||
{
|
||||
acceptNode(node) {
|
||||
return isSkippableTextNode(node as Text)
|
||||
? NodeFilter.FILTER_REJECT
|
||||
: NodeFilter.FILTER_ACCEPT;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const textNodes: Text[] = [];
|
||||
let currentNode = walker.nextNode();
|
||||
while (currentNode) {
|
||||
textNodes.push(currentNode as Text);
|
||||
currentNode = walker.nextNode();
|
||||
}
|
||||
|
||||
if (textNodes.length === 0) {
|
||||
return sanitizeTranslatedHtml(documentFragment.body.innerHTML);
|
||||
}
|
||||
|
||||
for (const batch of buildTextBatches(textNodes)) {
|
||||
const originalValues = batch.map((node) => node.textContent ?? "");
|
||||
const { payload } = createBatchPayload(originalValues);
|
||||
const translatedPayload = await params.translateText(payload, {
|
||||
protectedTerms: params.protectedTerms,
|
||||
isHtml: true,
|
||||
});
|
||||
|
||||
const translatedValues = parseBatchResponse(translatedPayload, originalValues);
|
||||
|
||||
batch.forEach((node, index) => {
|
||||
node.textContent = translatedValues[index];
|
||||
});
|
||||
}
|
||||
|
||||
return sanitizeTranslatedHtml(documentFragment.body.innerHTML);
|
||||
}
|
||||
Reference in New Issue
Block a user