office_translator/wordly.art---traduction-de-documents/src/utils/scriptDetector.ts

/**
 * L0 quality detector — TypeScript mirror of services/quality/ in Python.
 *
 * Detects:
 *   - wrong_script: translation is in the wrong script for the target language
 *     (e.g. user asks French, model returns Arabic — language confusion)
 *   - wrong_arabic_variant: for Arabic-script targets (ar/fa/ur/...),
 *     the translation uses characters of a DIFFERENT Arabic-script language
 *     (e.g. user asks Persian, model returns Urdu)
 *   - length_outlier / truncation_suspect: translation is wildly different
 *     in length from the source
 *   - prompt_leak: translation starts with "Translation:" or similar
 *   - repetition_hallucination: "xxx..." or "the the the the" pattern
 *
 * Zero dependencies. Works in browser AND in Node.js (for tests).
 * Designed to be ADDITIVE — never blocks the translation flow.
 */

// ---------- Types ----------

export interface QualityCheckResult {
  passed: boolean;
  score: number;             // 0.0 to 1.0
  issues: string[];
  detectedScript: string | null;
  expectedScript: string | null;
  details: Record<string, unknown>;
}

export interface DocumentQualityResult {
  passed: boolean;
  score: number;
  chunkCount: number;
  failedChunkCount: number;
  issues: Record<string, number>;
  samples: Array<{
    index: number;
    issues: string[];
    sourcePreview: string;
    translatedPreview: string;
    details: Record<string, unknown>;
  }>;
}

// ---------- Unicode ranges per script ----------
// Mirrors services/quality/config.py in Python.

type Range = readonly [number, number]; // [start, end] inclusive

const UNICODE_RANGES: Record<string, readonly Range[]> = {
  cyrillic: [
    [0x0400, 0x04ff],
    [0x0500, 0x052f],
  ],
  greek: [
    [0x0370, 0x03ff],
  ],
  arabic: [
    [0x0600, 0x06ff],
    [0x0750, 0x077f],
    [0x08a0, 0x08ff],
  ],
  hebrew: [
    [0x0590, 0x05ff],
  ],
  devanagari: [
    [0x0900, 0x097f],
  ],
  bengali: [
    [0x0980, 0x09ff],
  ],
  tamil: [
    [0x0b80, 0x0bff],
  ],
  telugu: [
    [0x0c00, 0x0c7f],
  ],
  kannada: [
    [0x0c80, 0x0cff],
  ],
  malayalam: [
    [0x0d00, 0x0d7f],
  ],
  sinhala: [
    [0x0d80, 0x0dff],
  ],
  gujarati: [
    [0x0a80, 0x0aff],
  ],
  gurmukhi: [
    [0x0a00, 0x0a7f],
  ],
  thai: [
    [0x0e00, 0x0e7f],
  ],
  lao: [
    [0x0e80, 0x0eff],
  ],
  burmese: [
    [0x1000, 0x109f],
  ],
  khmer: [
    [0x1780, 0x17ff],
  ],
  cjk: [
    [0x4e00, 0x9fff],
    [0x3400, 0x4dbf],
  ],
  hiragana_katakana: [
    [0x3040, 0x309f],
    [0x30a0, 0x30ff],
  ],
  hangul: [
    [0xac00, 0xd7af],
    [0x1100, 0x11ff],
    [0xa960, 0xa97f],
  ],
  georgian: [
    [0x10a0, 0x10ff],
  ],
  armenian: [
    [0x0530, 0x058f],
  ],
  ethiopic: [
    [0x1200, 0x137f],
    [0x1380, 0x139f],
  ],
  tibetan: [
    [0x0f00, 0x0fff],
  ],
  thaana: [
    [0x0780, 0x07bf],
  ],
  latin: [],
};

// ---------- Language → script mapping ----------

const LANG_TO_SCRIPT: Record<string, string> = {
  // Latin-script languages
  en: 'latin', fr: 'latin', de: 'latin', es: 'latin', it: 'latin',
  pt: 'latin', nl: 'latin', pl: 'latin', tr: 'latin', vi: 'latin',
  id: 'latin', ms: 'latin', ro: 'latin', cs: 'latin', sv: 'latin',
  da: 'latin', fi: 'latin', no: 'latin', nb: 'latin', nn: 'latin',
  hu: 'latin', sk: 'latin', sl: 'latin', lt: 'latin', lv: 'latin',
  et: 'latin', sq: 'latin', az: 'latin', uz: 'latin', kk: 'latin',
  ky: 'latin', tk: 'latin', sw: 'latin', eu: 'latin', gl: 'latin',
  is: 'latin', ga: 'latin', mt: 'latin', ca: 'latin', hr: 'latin',
  bs: 'latin', af: 'latin', cy: 'latin', lb: 'latin', fo: 'latin',
  br: 'latin', co: 'latin', fy: 'latin', gd: 'latin', gu: 'latin',
  ht: 'latin', haw: 'latin', hmn: 'latin', jv: 'latin', ku: 'latin',
  mg: 'latin', mi: 'latin', mn: 'latin', nso: 'latin', ny: 'latin',
  oc: 'latin', os: 'latin', ps: 'latin', qu: 'latin', rw: 'latin',
  sc: 'latin', si: 'latin', sm: 'latin', sn: 'latin', so: 'latin',
  st: 'latin', su: 'latin', tg: 'latin', tt: 'latin', ty: 'latin',
  ug: 'latin', vo: 'latin', wa: 'latin', wo: 'latin', xh: 'latin',
  yi: 'latin', zu: 'latin',
  // Cyrillic overrides
  ru: 'cyrillic', uk: 'cyrillic', be: 'cyrillic', sr: 'cyrillic',
  mk: 'cyrillic', bg: 'cyrillic', ce: 'cyrillic', cv: 'cyrillic',
  sah: 'cyrillic', udm: 'cyrillic', rue: 'cyrillic', ab: 'cyrillic',
  // Greek
  el: 'greek',
  // Arabic-script languages
  ar: 'arabic', fa: 'arabic', ur: 'arabic', ps: 'arabic',
  ku: 'arabic', sd: 'arabic', ckb: 'arabic', bal: 'arabic',
  bqi: 'arabic', glk: 'arabic', mzn: 'arabic',
  // Hebrew & Yiddish
  he: 'hebrew',
  yi: 'hebrew',   // Yiddish uses Hebrew script, not Arabic
  // Thaana (Maldivian)
  dv: 'thaana',   // Maldivian uses Thaana, not Arabic
  // Indian scripts
  hi: 'devanagari', ne: 'devanagari', mr: 'devanagari', sa: 'devanagari',
  mai: 'devanagari', bho: 'devanagari', awa: 'devanagari',
  bn: 'bengali', as: 'bengali',
  ta: 'tamil',
  te: 'telugu',
  kn: 'kannada',
  ml: 'malayalam',
  si: 'sinhala',
  gu: 'gujarati',
  pa: 'gurmukhi',
  // SE Asia
  th: 'thai',
  lo: 'lao',
  my: 'burmese',
  km: 'khmer',
  // East Asia
  zh: 'cjk', 'zh-cn': 'cjk', 'zh-tw': 'cjk', 'zh-hk': 'cjk',
  ja: 'hiragana_katakana',
  ko: 'hangul',
  // Others
  ka: 'georgian',
  hy: 'armenian',
  am: 'ethiopic', ti: 'ethiopic', gez: 'ethiopic', tig: 'ethiopic',
  bo: 'tibetan', dz: 'tibetan',
};

// ---------- Discriminating characters for Arabic-script languages ----------

const DISCRIMINATING_CHARS: Record<string, ReadonlySet<string>> = {
  fa: new Set('پچژگ'),                  // Persian
  ur: new Set('ٹڈڑے'),                  // Urdu
  ps: new Set('ټډړږښ'),                  // Pashto
  ku: new Set('ڕێ'),                     // Kurdish
  ckb: new Set('ڕێ'),                    // Sorani Kurdish
  sd: new Set('ٿ'),                     // Sindhi
  ug: new Set('ۇۆې'),                    // Uyghur
  bal: new Set('ێ'),                    // Balochi
};

// ---------- Thresholds ----------

const MIN_RATIO_IN_SCRIPT = 0.60;

// ---------- Helpers ----------

/** Get the script id for a language code. */
export function getScript(langCode: string | null | undefined): string {
  if (!langCode) return 'latin';
  return LANG_TO_SCRIPT[langCode.toLowerCase()] ?? 'latin';
}

/** True if the language uses an Arabic-script block. */
export function isArabicScriptLang(langCode: string | null | undefined): boolean {
  return getScript(langCode) === 'arabic';
}

/** Get the Unicode ranges for a script id. Empty array for 'latin'. */
function getRanges(scriptId: string): readonly Range[] {
  return UNICODE_RANGES[scriptId] ?? [];
}

/** Iterate code points of a string (handles surrogate pairs correctly). */
function* codePoints(text: string): Generator<number> {
  for (const ch of text) {
    yield ch.codePointAt(0) ?? 0;
  }
}

/** Check if a code point falls in any of the (start, end) ranges. */
function isInRanges(code: number, ranges: readonly Range[]): boolean {
  for (const [start, end] of ranges) {
    if (code >= start && code <= end) return true;
  }
  return false;
}

const LETTER_REGEX = /\p{L}/u;

/** Count alphabetic characters (using Unicode property escapes). */
function countLetters(text: string): number {
  let count = 0;
  for (const ch of text) {
    if (LETTER_REGEX.test(ch)) count++;
  }
  return count;
}

/** Count how many alphabetic characters fall within the given ranges. */
function countInScript(text: string, ranges: readonly Range[]): number {
  if (ranges.length === 0) {
    // Latin / unknown — all letters match.
    return countLetters(text);
  }
  let count = 0;
  for (const ch of text) {
    if (!LETTER_REGEX.test(ch)) continue;
    const cp = ch.codePointAt(0) ?? 0;
    if (isInRanges(cp, ranges)) count++;
  }
  return count;
}

// ---------- Arabic-script variant detection ----------

export interface ArabicVariantResult {
  verdict: 'pass' | 'fail' | 'skip';
  claimedLang: string | null;
  detectedVariants: string[];
  arabicRatio?: number;
  reason: string;
}

export function detectArabicVariant(
  text: string,
  claimedLang: string | null,
): ArabicVariantResult {
  if (!text || !text.trim()) {
    return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'empty text' };
  }

  const arabicRanges = getRanges('arabic');
  const letters = countLetters(text);
  if (letters === 0) {
    return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'no letters' };
  }

  const inArabic = countInScript(text, arabicRanges);
  const arabicRatio = inArabic / letters;

  if (arabicRatio < MIN_RATIO_IN_SCRIPT) {
    return {
      verdict: 'skip',
      claimedLang,
      detectedVariants: [],
      arabicRatio: round(arabicRatio, 3),
      reason: 'not in Arabic script',
    };
  }

  if (!isArabicScriptLang(claimedLang)) {
    return {
      verdict: 'skip',
      claimedLang,
      detectedVariants: [],
      arabicRatio: round(arabicRatio, 3),
      reason: 'target is not an Arabic-script language',
    };
  }

  // Text is Arabic-script AND target is Arabic-script: check the variant.
  const detected = new Set<string>();
  for (const [langCode, chars] of Object.entries(DISCRIMINATING_CHARS)) {
    if (chars.size === 0) continue;
    for (const ch of text) {
      if (chars.has(ch)) {
        detected.add(langCode);
        break;
      }
    }
  }

  if (detected.size > 0 && claimedLang && !detected.has(claimedLang.toLowerCase())) {
    return {
      verdict: 'fail',
      claimedLang,
      detectedVariants: Array.from(detected).sort(),
      arabicRatio: round(arabicRatio, 3),
      reason: `target=${claimedLang} but text contains characters typical of ${Array.from(detected).sort().join(', ')}`,
    };
  }

  return {
    verdict: 'pass',
    claimedLang,
    detectedVariants: detected.size > 0 ? Array.from(detected).sort() : (claimedLang ? [claimedLang] : []),
    arabicRatio: round(arabicRatio, 3),
    reason: 'ok',
  };
}

// ---------- Length check ----------

const RATIO_MAX = 3.5;
const RATIO_MIN = 0.15;
const ABSOLUTE_MIN_LENGTH = 2;
const MIN_SOURCE_LENGTH_FOR_RATIO = 20;

export interface LengthCheckResult {
  issue: string | null;
  ratio: number | null;
  sourceLength: number;
  translatedLength: number;
  note?: string;
}

export function lengthCheck(sourceText: string, translatedText: string): LengthCheckResult {
  if (!sourceText) {
    return {
      issue: null,
      ratio: null,
      sourceLength: 0,
      translatedLength: (translatedText || '').length,
    };
  }

  const srcLen = sourceText.trim().length;
  const transLen = (translatedText || '').trim().length;

  if (transLen === 0) {
    return {
      issue: 'truncation_suspect',
      ratio: 0,
      sourceLength: srcLen,
      translatedLength: transLen,
    };
  }

  if (isMostlyNumeric(sourceText)) {
    return {
      issue: null,
      ratio: null,
      sourceLength: srcLen,
      translatedLength: transLen,
      note: 'skipped: numeric source',
    };
  }

  if (srcLen < MIN_SOURCE_LENGTH_FOR_RATIO) {
    return {
      issue: null,
      ratio: null,
      sourceLength: srcLen,
      translatedLength: transLen,
    };
  }

  const ratio = transLen / srcLen;

  if (ratio > RATIO_MAX) {
    return {
      issue: 'length_outlier',
      ratio: round(ratio, 2),
      sourceLength: srcLen,
      translatedLength: transLen,
    };
  }
  if (ratio < RATIO_MIN) {
    return {
      issue: 'truncation_suspect',
      ratio: round(ratio, 2),
      sourceLength: srcLen,
      translatedLength: transLen,
    };
  }

  return {
    issue: null,
    ratio: round(ratio, 2),
    sourceLength: srcLen,
    translatedLength: transLen,
  };
}

function isMostlyNumeric(text: string): boolean {
  if (!text) return false;
  const chars: string[] = [];
  for (const ch of text) {
    if (!/\s/.test(ch)) chars.push(ch);
  }
  if (chars.length === 0) return false;
  const digitCount = chars.filter((c) => /\d/.test(c)).length;
  return digitCount / chars.length >= 0.5;
}

// ---------- Pattern leak / repetition ----------

const LEAK_PREFIX_PATTERNS: RegExp[] = [
  /^(translation|translated text|here is the translation|here'?s the translation)\s*[:：-]/i,
  /^(voici (la |ma )?traduction|traduction\s*[:：-])\b/i,
  /^(原文|译|翻译|译为|以下是)\s*[:：]?/u,
  /^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)/i,
  /^(\*\*|__|#)\s*translation/i,
  /^translated from\s+\w+\s+to\s+\w+\s*[:：-]/i,
];

const REPETITION_THRESHOLD = 5;
const CHAR_REPETITION_THRESHOLD = 20;

export interface PatternCheckResult {
  issue: string | null;
  matchedPattern: string | null;
  repetitionCount: number | null;
}

export function patternCheck(text: string): PatternCheckResult {
  if (!text || !text.trim()) {
    return { issue: null, matchedPattern: null, repetitionCount: null };
  }

  const stripped = text.trimStart();

  // 1. Prompt leak
  for (const pat of LEAK_PREFIX_PATTERNS) {
    if (pat.test(stripped)) {
      return {
        issue: 'prompt_leak',
        matchedPattern: pat.source,
        repetitionCount: null,
      };
    }
  }

  // 2. Token-level repetition
  const tokens = stripped.split(/\s+/).filter((t) => t.length > 0);
  const tokenRep = maxConsecutiveTokenRepetition(tokens);
  if (tokenRep >= REPETITION_THRESHOLD) {
    return {
      issue: 'repetition_hallucination',
      matchedPattern: null,
      repetitionCount: tokenRep,
    };
  }

  // 3. Character-level repetition
  const charRep = maxConsecutiveCharRepetition(stripped);
  if (charRep >= CHAR_REPETITION_THRESHOLD) {
    return {
      issue: 'repetition_hallucination',
      matchedPattern: null,
      repetitionCount: charRep,
    };
  }

  return {
    issue: null,
    matchedPattern: null,
    repetitionCount: Math.max(tokenRep, charRep) || null,
  };
}

function maxConsecutiveTokenRepetition(tokens: string[]): number {
  if (tokens.length === 0) return 0;
  const norm = tokens.map((t) => t.toLowerCase().replace(/[.,!?;:"'`()\[\]{}]/g, ''));
  let maxRun = 1;
  let currentRun = 1;
  for (let i = 1; i < norm.length; i++) {
    if (norm[i] && norm[i] === norm[i - 1]) {
      currentRun++;
      if (currentRun > maxRun) maxRun = currentRun;
    } else {
      currentRun = 1;
    }
  }
  return maxRun;
}

function maxConsecutiveCharRepetition(text: string): number {
  if (!text) return 0;
  let maxRun = 1;
  let currentRun = 1;
  for (let i = 1; i < text.length; i++) {
    if (text[i] === text[i - 1] && !/\s/.test(text[i])) {
      currentRun++;
      if (currentRun > maxRun) maxRun = currentRun;
    } else {
      currentRun = 1;
    }
  }
  return maxRun;
}

// ---------- Heuristic actual-script detection (for diagnostics) ----------

const SCRIPT_DETECTION_ORDER: readonly string[] = [
  'hiragana_katakana', 'hangul', 'cjk', 'thai', 'lao', 'burmese',
  'khmer', 'devanagari', 'bengali', 'tamil', 'telugu', 'kannada',
  'malayalam', 'sinhala', 'gujarati', 'gurmukhi',
  'arabic', 'hebrew', 'cyrillic', 'greek', 'armenian', 'georgian',
  'ethiopic', 'tibetan', 'thaana',
];

function detectActualScript(text: string): string {
  const letters = countLetters(text);
  if (letters === 0) return 'unknown';
  for (const scriptId of SCRIPT_DETECTION_ORDER) {
    const ranges = getRanges(scriptId);
    const inScript = countInScript(text, ranges);
    if (inScript / letters > 0.4) return scriptId;
  }
  return 'latin';
}

// ---------- Main per-chunk evaluation ----------

export function evaluateChunk(
  sourceText: string,
  translatedText: string | null | undefined,
  targetLang: string | null | undefined,
): QualityCheckResult {
  if (translatedText === null || translatedText === undefined) {
    return {
      passed: true,
      score: 0,
      issues: ['empty_translation'],
      detectedScript: null,
      expectedScript: null,
      details: { reason: 'translation is null/undefined' },
    };
  }

  const text = translatedText.trim();
  if (!text) {
    return {
      passed: true,
      score: 0,
      issues: ['empty_translation'],
      detectedScript: null,
      expectedScript: null,
      details: { reason: 'translation is empty or whitespace-only' },
    };
  }

  const targetLangLower = (targetLang || '').toLowerCase() || null;
  const issues: string[] = [];
  const details: Record<string, unknown> = {};

  // --- Script detection ---
  const expectedScript = getScript(targetLangLower);
  const expectedRanges = getRanges(expectedScript);
  const letters = countLetters(text);

  let scriptScore: number;
  let detectedScript: string | null;

  if (letters === 0) {
    scriptScore = 1.0;
    detectedScript = expectedScript;
    details.script_check = 'skipped: no alphabetic characters';
  } else {
    detectedScript = detectActualScript(text);
    const inExpected = countInScript(text, expectedRanges);
    scriptScore = inExpected / letters;

    details.script_score = round(scriptScore, 3);
    details.letters_in_text = letters;
    details.letters_in_script = inExpected;
    details.detected_script = detectedScript;
    details.expected_script = expectedScript;
    details.min_ratio = MIN_RATIO_IN_SCRIPT;

    if (expectedScript !== 'latin' && expectedRanges.length > 0) {
      // Specific non-Latin target.
      if (scriptScore < MIN_RATIO_IN_SCRIPT) {
        issues.push('wrong_script');
        details.reason = `only ${Math.round(scriptScore * 100)}% of letters match ${expectedScript} script; text appears to be in ${detectedScript}`;
      }
    } else {
      // Latin target. If detected script is clearly non-Latin, fail.
      if (detectedScript && detectedScript !== 'latin' && detectedScript !== 'unknown') {
        const nonLatinRanges = getRanges(detectedScript);
        const inDetected = countInScript(text, nonLatinRanges);
        const nonLatinConfidence = inDetected / letters;
        if (nonLatinConfidence >= 0.7) {
          issues.push('wrong_script');
          details.reason = `target is Latin but ${Math.round(nonLatinConfidence * 100)}% of letters are in ${detectedScript} script — language confusion`;
        }
      }
    }
  }

  // --- Arabic-script variant detection ---
  if (isArabicScriptLang(targetLangLower)) {
    const variantResult = detectArabicVariant(text, targetLangLower);
    details.arabic_variant = variantResult;
    if (variantResult.verdict === 'fail') {
      issues.push('wrong_arabic_variant');
    }
  }

  // --- Length sanity ---
  const lengthResult = lengthCheck(sourceText, text);
  details.length = lengthResult;
  if (lengthResult.issue) {
    issues.push(lengthResult.issue);
  }

  // --- Pattern leak / repetition ---
  const leakResult = patternCheck(text);
  details.pattern_check = leakResult;
  if (leakResult.issue) {
    issues.push(leakResult.issue);
  }

  // --- Aggregate ---
  const passed = issues.length === 0;
  const nChecks = 3;
  const nFailed = issues.filter((issue) =>
    ['wrong_script', 'wrong_arabic_variant', 'length_outlier', 'truncation_suspect', 'prompt_leak', 'repetition_hallucination'].includes(issue),
  ).length;
  const score = Math.max(0, 1 - nFailed / nChecks);

  return {
    passed,
    score: round(score, 3),
    issues,
    detectedScript,
    expectedScript,
    details,
  };
}

// ---------- Document-level aggregation ----------

export function evaluateDocument(
  sourceChunks: string[],
  translatedChunks: string[],
  targetLang: string | null | undefined,
  sampleSize: number = 50,
): DocumentQualityResult {
  const n = Math.min(sourceChunks.length, translatedChunks.length);
  const chunkResults: QualityCheckResult[] = [];
  const issuesCount: Record<string, number> = {};
  const samples: DocumentQualityResult['samples'] = [];
  let scoreSum = 0;
  let failedCount = 0;

  for (let i = 0; i < n; i++) {
    const r = evaluateChunk(sourceChunks[i], translatedChunks[i], targetLang);
    chunkResults.push(r);
    scoreSum += r.score;
    for (const issue of r.issues) {
      issuesCount[issue] = (issuesCount[issue] || 0) + 1;
    }
    if (!r.passed) {
      failedCount++;
      if (samples.length < sampleSize) {
        samples.push({
          index: i,
          issues: r.issues,
          sourcePreview: (sourceChunks[i] || '').slice(0, 80),
          translatedPreview: (translatedChunks[i] || '').slice(0, 80),
          details: r.details,
        });
      }
    }
  }

  const meanScore = n > 0 ? scoreSum / n : 0;
  const passed = failedCount === 0;

  return {
    passed,
    score: round(meanScore, 3),
    chunkCount: n,
    failedChunkCount: failedCount,
    issues: issuesCount,
    samples,
  };
}

// ---------- Utilities ----------

function round(value: number, decimals: number): number {
  const factor = 10 ** decimals;
  return Math.round(value * factor) / factor;
}