/** * L0 quality detector — TypeScript mirror of services/quality/ in Python. * * Detects: * - wrong_script: translation is in the wrong script for the target language * (e.g. user asks French, model returns Arabic — language confusion) * - wrong_arabic_variant: for Arabic-script targets (ar/fa/ur/...), * the translation uses characters of a DIFFERENT Arabic-script language * (e.g. user asks Persian, model returns Urdu) * - length_outlier / truncation_suspect: translation is wildly different * in length from the source * - prompt_leak: translation starts with "Translation:" or similar * - repetition_hallucination: "xxx..." or "the the the the" pattern * * Zero dependencies. Works in browser AND in Node.js (for tests). * Designed to be ADDITIVE — never blocks the translation flow. */ // ---------- Types ---------- export interface QualityCheckResult { passed: boolean; score: number; // 0.0 to 1.0 issues: string[]; detectedScript: string | null; expectedScript: string | null; details: Record; } export interface DocumentQualityResult { passed: boolean; score: number; chunkCount: number; failedChunkCount: number; issues: Record; samples: Array<{ index: number; issues: string[]; sourcePreview: string; translatedPreview: string; details: Record; }>; } // ---------- Unicode ranges per script ---------- // Mirrors services/quality/config.py in Python. type Range = readonly [number, number]; // [start, end] inclusive const UNICODE_RANGES: Record = { cyrillic: [ [0x0400, 0x04ff], [0x0500, 0x052f], ], greek: [ [0x0370, 0x03ff], ], arabic: [ [0x0600, 0x06ff], [0x0750, 0x077f], [0x08a0, 0x08ff], ], hebrew: [ [0x0590, 0x05ff], ], devanagari: [ [0x0900, 0x097f], ], bengali: [ [0x0980, 0x09ff], ], tamil: [ [0x0b80, 0x0bff], ], telugu: [ [0x0c00, 0x0c7f], ], kannada: [ [0x0c80, 0x0cff], ], malayalam: [ [0x0d00, 0x0d7f], ], sinhala: [ [0x0d80, 0x0dff], ], gujarati: [ [0x0a80, 0x0aff], ], gurmukhi: [ [0x0a00, 0x0a7f], ], thai: [ [0x0e00, 0x0e7f], ], lao: [ [0x0e80, 0x0eff], ], burmese: [ [0x1000, 0x109f], ], khmer: [ [0x1780, 0x17ff], ], cjk: [ [0x4e00, 0x9fff], [0x3400, 0x4dbf], ], hiragana_katakana: [ [0x3040, 0x309f], [0x30a0, 0x30ff], ], hangul: [ [0xac00, 0xd7af], [0x1100, 0x11ff], [0xa960, 0xa97f], ], georgian: [ [0x10a0, 0x10ff], ], armenian: [ [0x0530, 0x058f], ], ethiopic: [ [0x1200, 0x137f], [0x1380, 0x139f], ], tibetan: [ [0x0f00, 0x0fff], ], thaana: [ [0x0780, 0x07bf], ], latin: [], }; // ---------- Language → script mapping ---------- const LANG_TO_SCRIPT: Record = { // Latin-script languages en: 'latin', fr: 'latin', de: 'latin', es: 'latin', it: 'latin', pt: 'latin', nl: 'latin', pl: 'latin', tr: 'latin', vi: 'latin', id: 'latin', ms: 'latin', ro: 'latin', cs: 'latin', sv: 'latin', da: 'latin', fi: 'latin', no: 'latin', nb: 'latin', nn: 'latin', hu: 'latin', sk: 'latin', sl: 'latin', lt: 'latin', lv: 'latin', et: 'latin', sq: 'latin', az: 'latin', uz: 'latin', kk: 'latin', ky: 'latin', tk: 'latin', sw: 'latin', eu: 'latin', gl: 'latin', is: 'latin', ga: 'latin', mt: 'latin', ca: 'latin', hr: 'latin', bs: 'latin', af: 'latin', cy: 'latin', lb: 'latin', fo: 'latin', br: 'latin', co: 'latin', fy: 'latin', gd: 'latin', gu: 'latin', ht: 'latin', haw: 'latin', hmn: 'latin', jv: 'latin', ku: 'latin', mg: 'latin', mi: 'latin', mn: 'latin', nso: 'latin', ny: 'latin', oc: 'latin', os: 'latin', ps: 'latin', qu: 'latin', rw: 'latin', sc: 'latin', si: 'latin', sm: 'latin', sn: 'latin', so: 'latin', st: 'latin', su: 'latin', tg: 'latin', tt: 'latin', ty: 'latin', ug: 'latin', vo: 'latin', wa: 'latin', wo: 'latin', xh: 'latin', yi: 'latin', zu: 'latin', // Cyrillic overrides ru: 'cyrillic', uk: 'cyrillic', be: 'cyrillic', sr: 'cyrillic', mk: 'cyrillic', bg: 'cyrillic', ce: 'cyrillic', cv: 'cyrillic', sah: 'cyrillic', udm: 'cyrillic', rue: 'cyrillic', ab: 'cyrillic', // Greek el: 'greek', // Arabic-script languages ar: 'arabic', fa: 'arabic', ur: 'arabic', ps: 'arabic', ku: 'arabic', sd: 'arabic', ckb: 'arabic', bal: 'arabic', bqi: 'arabic', glk: 'arabic', mzn: 'arabic', // Hebrew & Yiddish he: 'hebrew', yi: 'hebrew', // Yiddish uses Hebrew script, not Arabic // Thaana (Maldivian) dv: 'thaana', // Maldivian uses Thaana, not Arabic // Indian scripts hi: 'devanagari', ne: 'devanagari', mr: 'devanagari', sa: 'devanagari', mai: 'devanagari', bho: 'devanagari', awa: 'devanagari', bn: 'bengali', as: 'bengali', ta: 'tamil', te: 'telugu', kn: 'kannada', ml: 'malayalam', si: 'sinhala', gu: 'gujarati', pa: 'gurmukhi', // SE Asia th: 'thai', lo: 'lao', my: 'burmese', km: 'khmer', // East Asia zh: 'cjk', 'zh-cn': 'cjk', 'zh-tw': 'cjk', 'zh-hk': 'cjk', ja: 'hiragana_katakana', ko: 'hangul', // Others ka: 'georgian', hy: 'armenian', am: 'ethiopic', ti: 'ethiopic', gez: 'ethiopic', tig: 'ethiopic', bo: 'tibetan', dz: 'tibetan', }; // ---------- Discriminating characters for Arabic-script languages ---------- const DISCRIMINATING_CHARS: Record> = { fa: new Set('پچژگ'), // Persian ur: new Set('ٹڈڑے'), // Urdu ps: new Set('ټډړږښ'), // Pashto ku: new Set('ڕێ'), // Kurdish ckb: new Set('ڕێ'), // Sorani Kurdish sd: new Set('ٿ'), // Sindhi ug: new Set('ۇۆې'), // Uyghur bal: new Set('ێ'), // Balochi }; // ---------- Thresholds ---------- const MIN_RATIO_IN_SCRIPT = 0.60; // ---------- Helpers ---------- /** Get the script id for a language code. */ export function getScript(langCode: string | null | undefined): string { if (!langCode) return 'latin'; return LANG_TO_SCRIPT[langCode.toLowerCase()] ?? 'latin'; } /** True if the language uses an Arabic-script block. */ export function isArabicScriptLang(langCode: string | null | undefined): boolean { return getScript(langCode) === 'arabic'; } /** Get the Unicode ranges for a script id. Empty array for 'latin'. */ function getRanges(scriptId: string): readonly Range[] { return UNICODE_RANGES[scriptId] ?? []; } /** Iterate code points of a string (handles surrogate pairs correctly). */ function* codePoints(text: string): Generator { for (const ch of text) { yield ch.codePointAt(0) ?? 0; } } /** Check if a code point falls in any of the (start, end) ranges. */ function isInRanges(code: number, ranges: readonly Range[]): boolean { for (const [start, end] of ranges) { if (code >= start && code <= end) return true; } return false; } const LETTER_REGEX = /\p{L}/u; /** Count alphabetic characters (using Unicode property escapes). */ function countLetters(text: string): number { let count = 0; for (const ch of text) { if (LETTER_REGEX.test(ch)) count++; } return count; } /** Count how many alphabetic characters fall within the given ranges. */ function countInScript(text: string, ranges: readonly Range[]): number { if (ranges.length === 0) { // Latin / unknown — all letters match. return countLetters(text); } let count = 0; for (const ch of text) { if (!LETTER_REGEX.test(ch)) continue; const cp = ch.codePointAt(0) ?? 0; if (isInRanges(cp, ranges)) count++; } return count; } // ---------- Arabic-script variant detection ---------- export interface ArabicVariantResult { verdict: 'pass' | 'fail' | 'skip'; claimedLang: string | null; detectedVariants: string[]; arabicRatio?: number; reason: string; } export function detectArabicVariant( text: string, claimedLang: string | null, ): ArabicVariantResult { if (!text || !text.trim()) { return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'empty text' }; } const arabicRanges = getRanges('arabic'); const letters = countLetters(text); if (letters === 0) { return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'no letters' }; } const inArabic = countInScript(text, arabicRanges); const arabicRatio = inArabic / letters; if (arabicRatio < MIN_RATIO_IN_SCRIPT) { return { verdict: 'skip', claimedLang, detectedVariants: [], arabicRatio: round(arabicRatio, 3), reason: 'not in Arabic script', }; } if (!isArabicScriptLang(claimedLang)) { return { verdict: 'skip', claimedLang, detectedVariants: [], arabicRatio: round(arabicRatio, 3), reason: 'target is not an Arabic-script language', }; } // Text is Arabic-script AND target is Arabic-script: check the variant. const detected = new Set(); for (const [langCode, chars] of Object.entries(DISCRIMINATING_CHARS)) { if (chars.size === 0) continue; for (const ch of text) { if (chars.has(ch)) { detected.add(langCode); break; } } } if (detected.size > 0 && claimedLang && !detected.has(claimedLang.toLowerCase())) { return { verdict: 'fail', claimedLang, detectedVariants: Array.from(detected).sort(), arabicRatio: round(arabicRatio, 3), reason: `target=${claimedLang} but text contains characters typical of ${Array.from(detected).sort().join(', ')}`, }; } return { verdict: 'pass', claimedLang, detectedVariants: detected.size > 0 ? Array.from(detected).sort() : (claimedLang ? [claimedLang] : []), arabicRatio: round(arabicRatio, 3), reason: 'ok', }; } // ---------- Length check ---------- const RATIO_MAX = 3.5; const RATIO_MIN = 0.15; const ABSOLUTE_MIN_LENGTH = 2; const MIN_SOURCE_LENGTH_FOR_RATIO = 20; export interface LengthCheckResult { issue: string | null; ratio: number | null; sourceLength: number; translatedLength: number; note?: string; } export function lengthCheck(sourceText: string, translatedText: string): LengthCheckResult { if (!sourceText) { return { issue: null, ratio: null, sourceLength: 0, translatedLength: (translatedText || '').length, }; } const srcLen = sourceText.trim().length; const transLen = (translatedText || '').trim().length; if (transLen === 0) { return { issue: 'truncation_suspect', ratio: 0, sourceLength: srcLen, translatedLength: transLen, }; } if (isMostlyNumeric(sourceText)) { return { issue: null, ratio: null, sourceLength: srcLen, translatedLength: transLen, note: 'skipped: numeric source', }; } if (srcLen < MIN_SOURCE_LENGTH_FOR_RATIO) { return { issue: null, ratio: null, sourceLength: srcLen, translatedLength: transLen, }; } const ratio = transLen / srcLen; if (ratio > RATIO_MAX) { return { issue: 'length_outlier', ratio: round(ratio, 2), sourceLength: srcLen, translatedLength: transLen, }; } if (ratio < RATIO_MIN) { return { issue: 'truncation_suspect', ratio: round(ratio, 2), sourceLength: srcLen, translatedLength: transLen, }; } return { issue: null, ratio: round(ratio, 2), sourceLength: srcLen, translatedLength: transLen, }; } function isMostlyNumeric(text: string): boolean { if (!text) return false; const chars: string[] = []; for (const ch of text) { if (!/\s/.test(ch)) chars.push(ch); } if (chars.length === 0) return false; const digitCount = chars.filter((c) => /\d/.test(c)).length; return digitCount / chars.length >= 0.5; } // ---------- Pattern leak / repetition ---------- const LEAK_PREFIX_PATTERNS: RegExp[] = [ /^(translation|translated text|here is the translation|here'?s the translation)\s*[::-]/i, /^(voici (la |ma )?traduction|traduction\s*[::-])\b/i, /^(原文|译|翻译|译为|以下是)\s*[::]?/u, /^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)/i, /^(\*\*|__|#)\s*translation/i, /^translated from\s+\w+\s+to\s+\w+\s*[::-]/i, ]; const REPETITION_THRESHOLD = 5; const CHAR_REPETITION_THRESHOLD = 20; export interface PatternCheckResult { issue: string | null; matchedPattern: string | null; repetitionCount: number | null; } export function patternCheck(text: string): PatternCheckResult { if (!text || !text.trim()) { return { issue: null, matchedPattern: null, repetitionCount: null }; } const stripped = text.trimStart(); // 1. Prompt leak for (const pat of LEAK_PREFIX_PATTERNS) { if (pat.test(stripped)) { return { issue: 'prompt_leak', matchedPattern: pat.source, repetitionCount: null, }; } } // 2. Token-level repetition const tokens = stripped.split(/\s+/).filter((t) => t.length > 0); const tokenRep = maxConsecutiveTokenRepetition(tokens); if (tokenRep >= REPETITION_THRESHOLD) { return { issue: 'repetition_hallucination', matchedPattern: null, repetitionCount: tokenRep, }; } // 3. Character-level repetition const charRep = maxConsecutiveCharRepetition(stripped); if (charRep >= CHAR_REPETITION_THRESHOLD) { return { issue: 'repetition_hallucination', matchedPattern: null, repetitionCount: charRep, }; } return { issue: null, matchedPattern: null, repetitionCount: Math.max(tokenRep, charRep) || null, }; } function maxConsecutiveTokenRepetition(tokens: string[]): number { if (tokens.length === 0) return 0; const norm = tokens.map((t) => t.toLowerCase().replace(/[.,!?;:"'`()\[\]{}]/g, '')); let maxRun = 1; let currentRun = 1; for (let i = 1; i < norm.length; i++) { if (norm[i] && norm[i] === norm[i - 1]) { currentRun++; if (currentRun > maxRun) maxRun = currentRun; } else { currentRun = 1; } } return maxRun; } function maxConsecutiveCharRepetition(text: string): number { if (!text) return 0; let maxRun = 1; let currentRun = 1; for (let i = 1; i < text.length; i++) { if (text[i] === text[i - 1] && !/\s/.test(text[i])) { currentRun++; if (currentRun > maxRun) maxRun = currentRun; } else { currentRun = 1; } } return maxRun; } // ---------- Heuristic actual-script detection (for diagnostics) ---------- const SCRIPT_DETECTION_ORDER: readonly string[] = [ 'hiragana_katakana', 'hangul', 'cjk', 'thai', 'lao', 'burmese', 'khmer', 'devanagari', 'bengali', 'tamil', 'telugu', 'kannada', 'malayalam', 'sinhala', 'gujarati', 'gurmukhi', 'arabic', 'hebrew', 'cyrillic', 'greek', 'armenian', 'georgian', 'ethiopic', 'tibetan', 'thaana', ]; function detectActualScript(text: string): string { const letters = countLetters(text); if (letters === 0) return 'unknown'; for (const scriptId of SCRIPT_DETECTION_ORDER) { const ranges = getRanges(scriptId); const inScript = countInScript(text, ranges); if (inScript / letters > 0.4) return scriptId; } return 'latin'; } // ---------- Main per-chunk evaluation ---------- export function evaluateChunk( sourceText: string, translatedText: string | null | undefined, targetLang: string | null | undefined, ): QualityCheckResult { if (translatedText === null || translatedText === undefined) { return { passed: true, score: 0, issues: ['empty_translation'], detectedScript: null, expectedScript: null, details: { reason: 'translation is null/undefined' }, }; } const text = translatedText.trim(); if (!text) { return { passed: true, score: 0, issues: ['empty_translation'], detectedScript: null, expectedScript: null, details: { reason: 'translation is empty or whitespace-only' }, }; } const targetLangLower = (targetLang || '').toLowerCase() || null; const issues: string[] = []; const details: Record = {}; // --- Script detection --- const expectedScript = getScript(targetLangLower); const expectedRanges = getRanges(expectedScript); const letters = countLetters(text); let scriptScore: number; let detectedScript: string | null; if (letters === 0) { scriptScore = 1.0; detectedScript = expectedScript; details.script_check = 'skipped: no alphabetic characters'; } else { detectedScript = detectActualScript(text); const inExpected = countInScript(text, expectedRanges); scriptScore = inExpected / letters; details.script_score = round(scriptScore, 3); details.letters_in_text = letters; details.letters_in_script = inExpected; details.detected_script = detectedScript; details.expected_script = expectedScript; details.min_ratio = MIN_RATIO_IN_SCRIPT; if (expectedScript !== 'latin' && expectedRanges.length > 0) { // Specific non-Latin target. if (scriptScore < MIN_RATIO_IN_SCRIPT) { issues.push('wrong_script'); details.reason = `only ${Math.round(scriptScore * 100)}% of letters match ${expectedScript} script; text appears to be in ${detectedScript}`; } } else { // Latin target. If detected script is clearly non-Latin, fail. if (detectedScript && detectedScript !== 'latin' && detectedScript !== 'unknown') { const nonLatinRanges = getRanges(detectedScript); const inDetected = countInScript(text, nonLatinRanges); const nonLatinConfidence = inDetected / letters; if (nonLatinConfidence >= 0.7) { issues.push('wrong_script'); details.reason = `target is Latin but ${Math.round(nonLatinConfidence * 100)}% of letters are in ${detectedScript} script — language confusion`; } } } } // --- Arabic-script variant detection --- if (isArabicScriptLang(targetLangLower)) { const variantResult = detectArabicVariant(text, targetLangLower); details.arabic_variant = variantResult; if (variantResult.verdict === 'fail') { issues.push('wrong_arabic_variant'); } } // --- Length sanity --- const lengthResult = lengthCheck(sourceText, text); details.length = lengthResult; if (lengthResult.issue) { issues.push(lengthResult.issue); } // --- Pattern leak / repetition --- const leakResult = patternCheck(text); details.pattern_check = leakResult; if (leakResult.issue) { issues.push(leakResult.issue); } // --- Aggregate --- const passed = issues.length === 0; const nChecks = 3; const nFailed = issues.filter((issue) => ['wrong_script', 'wrong_arabic_variant', 'length_outlier', 'truncation_suspect', 'prompt_leak', 'repetition_hallucination'].includes(issue), ).length; const score = Math.max(0, 1 - nFailed / nChecks); return { passed, score: round(score, 3), issues, detectedScript, expectedScript, details, }; } // ---------- Document-level aggregation ---------- export function evaluateDocument( sourceChunks: string[], translatedChunks: string[], targetLang: string | null | undefined, sampleSize: number = 50, ): DocumentQualityResult { const n = Math.min(sourceChunks.length, translatedChunks.length); const chunkResults: QualityCheckResult[] = []; const issuesCount: Record = {}; const samples: DocumentQualityResult['samples'] = []; let scoreSum = 0; let failedCount = 0; for (let i = 0; i < n; i++) { const r = evaluateChunk(sourceChunks[i], translatedChunks[i], targetLang); chunkResults.push(r); scoreSum += r.score; for (const issue of r.issues) { issuesCount[issue] = (issuesCount[issue] || 0) + 1; } if (!r.passed) { failedCount++; if (samples.length < sampleSize) { samples.push({ index: i, issues: r.issues, sourcePreview: (sourceChunks[i] || '').slice(0, 80), translatedPreview: (translatedChunks[i] || '').slice(0, 80), details: r.details, }); } } } const meanScore = n > 0 ? scoreSum / n : 0; const passed = failedCount === 0; return { passed, score: round(meanScore, 3), chunkCount: n, failedChunkCount: failedCount, issues: issuesCount, samples, }; } // ---------- Utilities ---------- function round(value: number, decimals: number): number { const factor = 10 ** decimals; return Math.round(value * factor) / factor; }