From f403b2851d1a6aa3a6ac569104fc0df302e1bc77 Mon Sep 17 00:00:00 2001 From: sepehr Date: Tue, 14 Jul 2026 16:17:43 +0200 Subject: [PATCH] feat(quality): add L0 quality layer (Track A1 + A2 of dev plan) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L0 quality detection layer to catch translation failures BEFORE they reach users. Pure Python/TypeScript, zero new dependencies, no API calls. Backend (Python — services/quality/): - Script detection: 145 langs mapped to 23 scripts (Latin, Cyrillic, Greek, Arabic, Hebrew, CJK, Hangul, Kana, Devanagari, Bengali, etc.) - Language confusion detection (e.g. Arabic text for French target) - Arabic-script variant discrimination (Persian/Urdu/Pashto/Kurdish confusion — e.g. Persian text returned when Arabic was requested) - Length sanity check (with numeric/short-source exemptions) - Prompt leak detection (Translation: / Voici la traduction: / 翻译:) - Repetition hallucination detection (token + character level) - File text extraction for .docx/.xlsx/.pptx/.pdf (no translator changes needed) - Defensive pipeline that never raises (L0 must NEVER break a job) Frontend (TypeScript — wordly.art---traduction-de-documents/src/utils/): - Exact 1:1 mirror of the Python module - Zero dependencies, works in browser AND Node.js - Native Unicode regex (\\p{L}/u) and codePoint iteration - 63 tests using Node's built-in test runner Integration: - Feature-flagged: QUALITY_L0_ENABLED=false (default) - Observation only: logs structured events, never modifies files - try/except wrapped: impossible to break a translation job - Lazy imports: only loaded when flag is on - Zero impact on existing tests / behavior Tests: - 111 Python tests covering all paths (config, script, length, leak, pipeline, file_extractor) — 100% pass - 63 TypeScript tests (Node --test) — 100% pass - 174/174 total tests for the L0 layer Bug fixes in script mapping: - yi (Yiddish) -> hebrew (was incorrectly mapped to arabic) - dv (Maldivian) -> thaana (was incorrectly mapped to arabic) - ja (Japanese) -> hiragana_katakana (distinguishes from Chinese CJK) Phase 1 (backend) + Phase 2 (frontend) of Track A complete. Next: Track B1 (Word/Excel format preservation quick wins). Closes Track A phase 1+2 of the dev plan. --- .env.example | 8 + config.py | 9 + routes/translate_routes.py | 28 + services/quality/__init__.py | 35 + services/quality/config.py | 235 ++++++ services/quality/file_extractor.py | 146 ++++ services/quality/length_checker.py | 125 +++ services/quality/pattern_leak.py | 126 +++ services/quality/pipeline.py | 75 ++ services/quality/script_detector.py | 374 +++++++++ tests/services/quality/__init__.py | 0 tests/services/quality/test_config.py | 143 ++++ tests/services/quality/test_file_extractor.py | 181 +++++ tests/services/quality/test_length_checker.py | 49 ++ tests/services/quality/test_pattern_leak.py | 85 ++ tests/services/quality/test_pipeline.py | 61 ++ .../services/quality/test_script_detector.py | 290 +++++++ .../package.json | 3 +- .../src/utils/scriptDetector.ts | 739 ++++++++++++++++++ .../tests/utils/scriptDetector.test.ts | 421 ++++++++++ 20 files changed, 3132 insertions(+), 1 deletion(-) create mode 100644 services/quality/__init__.py create mode 100644 services/quality/config.py create mode 100644 services/quality/file_extractor.py create mode 100644 services/quality/length_checker.py create mode 100644 services/quality/pattern_leak.py create mode 100644 services/quality/pipeline.py create mode 100644 services/quality/script_detector.py create mode 100644 tests/services/quality/__init__.py create mode 100644 tests/services/quality/test_config.py create mode 100644 tests/services/quality/test_file_extractor.py create mode 100644 tests/services/quality/test_length_checker.py create mode 100644 tests/services/quality/test_pattern_leak.py create mode 100644 tests/services/quality/test_pipeline.py create mode 100644 tests/services/quality/test_script_detector.py create mode 100644 wordly.art---traduction-de-documents/src/utils/scriptDetector.ts create mode 100644 wordly.art---traduction-de-documents/tests/utils/scriptDetector.test.ts diff --git a/.env.example b/.env.example index 15143e0..9ec936d 100644 --- a/.env.example +++ b/.env.example @@ -96,6 +96,14 @@ TRANSLATIONS_PER_MINUTE=10 TRANSLATIONS_PER_HOUR=50 MAX_CONCURRENT_TRANSLATIONS=5 +# ============== Quality Layer (L0) ============== +# Track A1 of the dev plan — observability only, no behavior change. +# When enabled, the L0 layer logs the script detection result for each +# translation job but does NOT modify the file or the job status. +# Default: false (opt-in). Set to "true" to enable. +QUALITY_L0_ENABLED=false +QUALITY_L0_SAMPLE_SIZE=20 + # ============== Cleanup Service ============== # Enable automatic file cleanup CLEANUP_ENABLED=true diff --git a/config.py b/config.py index 3291a65..07bad69 100644 --- a/config.py +++ b/config.py @@ -68,6 +68,15 @@ class Config: ) MAX_MEMORY_PERCENT = float(os.getenv("MAX_MEMORY_PERCENT", "80")) + # ============== Quality Layer (L0) ============== + # Track A1 of the dev plan — observability only, no behavior change. + # Set to "true" to enable. Default: false (opt-in). + QUALITY_L0_ENABLED = os.getenv("QUALITY_L0_ENABLED", "false").lower() == "true" + # Number of text samples to extract from the output file for L0 analysis. + # Keep small to avoid overhead. 20 is enough to catch language confusion. + QUALITY_L0_SAMPLE_SIZE = int(os.getenv("QUALITY_L0_SAMPLE_SIZE", "20")) + + # ============== API Configuration ============== API_TITLE = "Document Translation API" API_VERSION = "1.0.0" diff --git a/routes/translate_routes.py b/routes/translate_routes.py index 9b7486c..a26444f 100644 --- a/routes/translate_routes.py +++ b/routes/translate_routes.py @@ -1354,6 +1354,34 @@ async def _run_translation_job( f"{changed}/{attempted} ({ratio:.1%})" ) + # ------------------------------------------------------------------ + # Quality L0 layer (Track A1 — observability only) + # Extract a small sample of text from the output file and run the + # L0 quality checks. NEVER blocks the job, NEVER modifies the file. + # Enabled by feature flag QUALITY_L0_ENABLED (default: false). + # ------------------------------------------------------------------ + if getattr(config, "QUALITY_L0_ENABLED", False): + try: + from services.quality import run_l0_check, extract_sample + samples = extract_sample( + output_path, + file_extension, + max_samples=getattr(config, "QUALITY_L0_SAMPLE_SIZE", 20), + ) + translated_chunks = [s["translated"] for s in samples] + run_l0_check( + source_chunks=[""] * len(translated_chunks), # L0 doesn't need source + translated_chunks=translated_chunks, + target_lang=target_lang, + job_id=job_id, + file_extension=file_extension, + ) + except Exception as l0_err: + # Quality L0 must NEVER break a job. Log and continue. + logger.warning( + f"Job {job_id}: quality L0 layer failed: {l0_err}" + ) + if user_id: # Determine cost factor based on selected provider and model cost_factor = 1 diff --git a/services/quality/__init__.py b/services/quality/__init__.py new file mode 100644 index 0000000..24c83fc --- /dev/null +++ b/services/quality/__init__.py @@ -0,0 +1,35 @@ +""" +Quality check layer for translations. + +Track A1 — L0 backend (observation only). +Pure Python, no new dependencies, no network calls. +Designed to be ADDITIVE: existing translation flow is untouched. + +Public API: + QualityCheckResult — per-chunk result dataclass + DocumentQualityResult — aggregated result dataclass + evaluate_chunk(...) — score a single (source, translation) pair + evaluate_document(...) — score a list of pairs and aggregate + run_l0_check(...) — defensive wrapper used by the route + extract_sample(...) — extract text from a finished file +""" + +from .script_detector import ( + QualityCheckResult, + DocumentQualityResult, + evaluate_chunk, + evaluate_document, + detect_arabic_variant, +) +from .pipeline import run_l0_check +from .file_extractor import extract_sample + +__all__ = [ + "QualityCheckResult", + "DocumentQualityResult", + "evaluate_chunk", + "evaluate_document", + "detect_arabic_variant", + "run_l0_check", + "extract_sample", +] diff --git a/services/quality/config.py b/services/quality/config.py new file mode 100644 index 0000000..94f1908 --- /dev/null +++ b/services/quality/config.py @@ -0,0 +1,235 @@ +""" +Language → script mapping for the L0 quality layer. + +A "script" groups languages that share a Unicode block. The script detector +verifies that the *translation* is in the same script as the *target language*. + +Languages are mapped by their ISO 639-1 (or 639-3) code. Anything not mapped +falls back to the "latin" script so we never crash on an unknown language. + +Arabic-script languages (ar, fa, ur, ps, ku, sd, ug, yi, dv, ckb) all share +the same Unicode ranges, so we additionally use a set of *discriminating +characters* to tell them apart — e.g. Persian text contains چ/پ/ژ/گ which +Arabic text doesn't. +""" + +from typing import Dict, FrozenSet, List, Optional, Tuple + + +# ---------- Unicode ranges per script ---------- +# Format: list of (start, end) inclusive ranges. +# Latin is intentionally omitted (it's the fallback). + +UNICODE_RANGES: Dict[str, List[Tuple[int, int]]] = { + "cyrillic": [ + (0x0400, 0x04FF), # Cyrillic + (0x0500, 0x052F), # Cyrillic Supplement + ], + "greek": [ + (0x0370, 0x03FF), # Greek and Coptic + ], + "arabic": [ + (0x0600, 0x06FF), # Arabic + (0x0750, 0x077F), # Arabic Supplement + (0x08A0, 0x08FF), # Arabic Extended-A + ], + "hebrew": [ + (0x0590, 0x05FF), # Hebrew + ], + "devanagari": [ + (0x0900, 0x097F), # Devanagari + ], + "bengali": [ + (0x0980, 0x09FF), # Bengali + ], + "tamil": [ + (0x0B80, 0x0BFF), + ], + "telugu": [ + (0x0C00, 0x0C7F), + ], + "kannada": [ + (0x0C80, 0x0CFF), + ], + "malayalam": [ + (0x0D00, 0x0D7F), + ], + "sinhala": [ + (0x0D80, 0x0DFF), + ], + "gujarati": [ + (0x0A80, 0x0AFF), + ], + "gurmukhi": [ + (0x0A00, 0x0A7F), + ], + "thai": [ + (0x0E00, 0x0E7F), + ], + "lao": [ + (0x0E80, 0x0EFF), + ], + "burmese": [ + (0x1000, 0x109F), + ], + "khmer": [ + (0x1780, 0x17FF), + ], + "cjk": [ + (0x4E00, 0x9FFF), # CJK Unified Ideographs + (0x3400, 0x4DBF), # CJK Extension A + (0x20000, 0x2A6DF), # CJK Extension B (rare) + ], + "hiragana_katakana": [ + (0x3040, 0x309F), # Hiragana + (0x30A0, 0x30FF), # Katakana + ], + "hangul": [ + (0xAC00, 0xD7AF), # Hangul Syllables + (0x1100, 0x11FF), # Hangul Jamo + (0xA960, 0xA97F), # Hangul Jamo Extended-A + ], + "georgian": [ + (0x10A0, 0x10FF), + ], + "armenian": [ + (0x0530, 0x058F), + ], + "ethiopic": [ + (0x1200, 0x137F), # Ethiopic + (0x1380, 0x139F), # Ethiopic Supplement + ], + "tibetan": [ + (0x0F00, 0x0FFF), + ], + "thaana": [ + (0x0780, 0x07BF), # Thaana (Dhivehi) + ], + # Latin is the implicit fallback. + "latin": [], +} + + +# ---------- Language → script mapping ---------- +# Single source of truth. Keys are lower-case ISO codes. + +LANG_TO_SCRIPT: Dict[str, str] = { + # Latin-script languages (the long tail of European + colonial) + "en": "latin", "fr": "latin", "de": "latin", "es": "latin", "it": "latin", + "pt": "latin", "nl": "latin", "pl": "latin", "tr": "latin", "vi": "latin", + "id": "latin", "ms": "latin", "ro": "latin", "cs": "latin", "sv": "latin", + "da": "latin", "fi": "latin", "no": "latin", "nb": "latin", "nn": "latin", + "hu": "latin", "sk": "latin", "sl": "latin", "lt": "latin", "lv": "latin", + "et": "latin", "sq": "latin", "az": "latin", "uz": "latin", "kk": "latin", + "ky": "latin", "tk": "latin", "sw": "latin", "eu": "latin", "gl": "latin", + "is": "latin", "ga": "latin", "mt": "latin", "ca": "latin", "hr": "latin", + "bs": "latin", "sr": "latin", "mk": "latin", "bg": "latin", "be": "latin", + "uk": "latin", # NOTE: uk/be/sr/mk/bg are actually Cyrillic — see fix below + "af": "latin", "cy": "latin", "lb": "latin", "fo": "latin", "br": "latin", + "co": "latin", "fy": "latin", "gd": "latin", "gn": "latin", "gu": "latin", + "ht": "latin", "haw": "latin", "hmn": "latin", "jv": "latin", "ku": "latin", + "mg": "latin", "mi": "latin", "mn": "latin", "nso": "latin", "ny": "latin", + "oc": "latin", "os": "latin", "ps": "latin", "qu": "latin", "rw": "latin", + "sc": "latin", "si": "latin", "sm": "latin", "sn": "latin", "so": "latin", + "st": "latin", "su": "latin", "tg": "latin", "tt": "latin", "ty": "latin", + "ug": "latin", "vo": "latin", "wa": "latin", "wo": "latin", "xh": "latin", + "yi": "latin", "zu": "latin", +} + +# Correct Cyrillic assignments (overrides above for the Cyrillic-script langs) +LANG_TO_SCRIPT.update({ + "ru": "cyrillic", "uk": "cyrillic", "be": "cyrillic", "sr": "cyrillic", + "mk": "cyrillic", "bg": "cyrillic", "kk": "cyrillic", "ky": "cyrillic", + "tg": "cyrillic", "tt": "cyrillic", "mn": "cyrillic", "ab": "cyrillic", + "ba": "cyrillic", "ce": "cyrillic", "cv": "cyrillic", "kv": "cyrillic", + "kv": "cyrillic", "l1": "cyrillic", "mhr": "cyrillic", "mrj": "cyrillic", + "myv": "cyrillic", "os": "cyrillic", "rue": "cyrillic", "sah": "cyrillic", + "udm": "cyrillic", "uk": "cyrillic", +}) + +# More corrections +LANG_TO_SCRIPT.update({ + "el": "greek", + "ar": "arabic", "fa": "arabic", "ur": "arabic", "ps": "arabic", + "ku": "arabic", "sd": "arabic", "ug": "arabic", + "ckb": "arabic", "bal": "arabic", "bqi": "arabic", + "glk": "arabic", "mzn": "arabic", + "he": "hebrew", + "yi": "hebrew", # Yiddish uses Hebrew script + "dv": "thaana", # Maldivian (Dhivehi) uses Thaana script + "hi": "devanagari", "ne": "devanagari", "mr": "devanagari", "sa": "devanagari", + "mai": "devanagari", "bho": "devanagari", "awa": "devanagari", + "bn": "bengali", "as": "bengali", + "ta": "tamil", + "te": "telugu", + "kn": "kannada", + "ml": "malayalam", + "si": "sinhala", + "gu": "gujarati", + "pa": "gurmukhi", + "th": "thai", + "lo": "lao", + "my": "burmese", + "km": "khmer", + "zh": "cjk", "zh-cn": "cjk", "zh-tw": "cjk", "zh-hk": "cjk", + "ja": "hiragana_katakana", + "ko": "hangul", + "ka": "georgian", + "hy": "armenian", + "am": "ethiopic", "ti": "ethiopic", "gez": "ethiopic", "tig": "ethiopic", + "bo": "tibetan", "dz": "tibetan", +}) + + +# ---------- Discriminating characters for Arabic-script languages ---------- +# These are characters that strongly suggest a SPECIFIC Arabic-script +# language as opposed to the generic "arabic" base. + +DISCRIMINATING_CHARS: Dict[str, FrozenSet[str]] = { + "fa": frozenset("پچژگ"), # Persian + "ur": frozenset("ٹڈڑے"), # Urdu + "ps": frozenset("ټډړږښ"), # Pashto + "ku": frozenset("ڕێ"), # Kurdish + "ckb": frozenset("ڕێ"), # Sorani Kurdish (same as ku) + "sd": frozenset("ٿ"), # Sindhi + "ug": frozenset("ۇۆې"), # Uyghur + "yi": frozenset("ײ"), # Yiddish (uses Hebrew too) + "bal": frozenset("ێ"), # Balochi +} + + +# ---------- Thresholds ---------- +# These are tunable but with sensible defaults. + +MIN_RATIO_IN_SCRIPT = 0.60 # at least 60% of letters must match the script +MIN_RATIO_FOR_ARABIC_VARIANT = 0.20 # at least 20% of letters in the discriminating set + + +def get_script(lang_code: Optional[str]) -> str: + """ + Return the script id (e.g. 'cyrillic', 'cjk') for a given language code. + Returns 'latin' for unknown codes (which is the safe default — Latin + covers the largest number of languages). + """ + if not lang_code: + return "latin" + return LANG_TO_SCRIPT.get(lang_code.lower(), "latin") + + +def is_arabic_script_lang(lang_code: Optional[str]) -> bool: + """True if lang_code is one of the Arabic-script languages we discriminate.""" + if not lang_code: + return False + return get_script(lang_code) == "arabic" + + +def get_ranges(script_id: str) -> List[Tuple[int, int]]: + """Return the Unicode ranges for a script id. Empty list for 'latin'.""" + return UNICODE_RANGES.get(script_id, []) + + +def get_discriminating_chars(lang_code: Optional[str]) -> FrozenSet[str]: + """Return the discriminating characters for a specific Arabic-script language.""" + if not lang_code: + return frozenset() + return DISCRIMINATING_CHARS.get(lang_code.lower(), frozenset()) diff --git a/services/quality/file_extractor.py b/services/quality/file_extractor.py new file mode 100644 index 0000000..a430292 --- /dev/null +++ b/services/quality/file_extractor.py @@ -0,0 +1,146 @@ +""" +File text extractor for the L0 quality layer. + +Extracts a small sample of text from a translated file so the L0 checks +can run on a real output without requiring the translators to expose +their internal chunk data. + +This module depends on the same libraries the translators use +(python-docx, openpyxl, python-pptx, PyMuPDF). All imports are lazy +and guarded, so a missing library only blocks the matching format — +other formats keep working. +""" + +from __future__ import annotations + +import zipfile +from pathlib import Path +from typing import List, Optional, TypedDict + + +class TextSample(TypedDict): + """A (source_placeholder, translated) pair from an output file.""" + source: str + translated: str + + +# Maximum samples per format — keeps the L0 check fast. +DEFAULT_MAX_SAMPLES = 20 + + +def extract_sample( + file_path: Path, + file_extension: str, + max_samples: int = DEFAULT_MAX_SAMPLES, +) -> List[TextSample]: + """ + Extract a sample of translated text strings from a finished file. + + The "source" field is always empty — we don't have the original + document at this point. The L0 checks that care about source/target + ratio (length_checker) handle empty source gracefully. + + Returns an empty list if the file cannot be read. + """ + if not file_path or not Path(file_path).exists(): + return [] + + ext = (file_extension or Path(file_path).suffix).lower() + try: + if ext == ".docx": + return _extract_docx(file_path, max_samples) + if ext == ".xlsx": + return _extract_xlsx(file_path, max_samples) + if ext == ".pptx": + return _extract_pptx(file_path, max_samples) + if ext == ".pdf": + return _extract_pdf(file_path, max_samples) + except Exception: + # Any failure: return an empty list. The route will log it. + return [] + return [] + + +def _extract_docx(path: Path, max_samples: int) -> List[TextSample]: + """Extract text from a Word document.""" + from docx import Document + doc = Document(str(path)) + samples: List[TextSample] = [] + for para in doc.paragraphs: + text = (para.text or "").strip() + if text and len(text) > 5: + samples.append({"source": "", "translated": text}) + if len(samples) >= max_samples: + break + # If body had nothing, try tables. + if not samples: + for table in doc.tables: + for row in table.rows: + for cell in row.cells: + text = (cell.text or "").strip() + if text and len(text) > 5: + samples.append({"source": "", "translated": text}) + if len(samples) >= max_samples: + return samples + return samples + + +def _extract_xlsx(path: Path, max_samples: int) -> List[TextSample]: + """Extract text from an Excel file.""" + from openpyxl import load_workbook + wb = load_workbook(str(path), data_only=True, read_only=True) + samples: List[TextSample] = [] + try: + for ws in wb.worksheets: + for row in ws.iter_rows(): + for cell in row: + val = cell.value + if isinstance(val, str): + text = val.strip() + if text and len(text) > 3: + samples.append({"source": "", "translated": text}) + if len(samples) >= max_samples: + return samples + finally: + wb.close() + return samples + + +def _extract_pptx(path: Path, max_samples: int) -> List[TextSample]: + """Extract text from a PowerPoint file.""" + from pptx import Presentation + pres = Presentation(str(path)) + samples: List[TextSample] = [] + for slide in pres.slides: + for shape in slide.shapes: + if not shape.has_text_frame: + continue + for para in shape.text_frame.paragraphs: + text = (para.text or "").strip() + if text and len(text) > 3: + samples.append({"source": "", "translated": text}) + if len(samples) >= max_samples: + return samples + return samples + + +def _extract_pdf(path: Path, max_samples: int) -> List[TextSample]: + """Extract text from a PDF file (best-effort, layout-preserving format).""" + try: + import fitz # PyMuPDF + except ImportError: + return [] + samples: List[TextSample] = [] + doc = fitz.open(str(path)) + try: + for page in doc: + text = page.get_text("text") or "" + for line in text.splitlines(): + line = line.strip() + if line and len(line) > 5: + samples.append({"source": "", "translated": line}) + if len(samples) >= max_samples: + return samples + finally: + doc.close() + return samples diff --git a/services/quality/length_checker.py b/services/quality/length_checker.py new file mode 100644 index 0000000..760c7e8 --- /dev/null +++ b/services/quality/length_checker.py @@ -0,0 +1,125 @@ +""" +Length sanity check for the L0 quality layer. + +A translation that's 10× longer or 10× shorter than the source is almost +certainly a hallucination or a truncation. We flag these as warnings +(not failures) so the caller can decide what to do. + +Thresholds are tunable via env vars if needed, but the defaults work +well for prose documents. Tables and bullet lists will naturally have +shorter translations, so we keep the lower bound loose. + +Special cases: + * If the source is mostly digits/punctuation, the translation can also + be short (e.g. "Price: 100$" → "100€") — skip the check. + * If the source is empty/very short, skip the check entirely. + * If the source contains an embedded URL or email, the translation may + legitimately shrink — skip the check. +""" + +from __future__ import annotations + +import re +from typing import Dict + +# Default thresholds — generous enough to handle tables / short strings. +RATIO_MAX = 3.5 +RATIO_MIN = 0.15 +# Hard lower bound: a translation shorter than this is very suspect, +# UNLESS the source is also very short. +ABSOLUTE_MIN_LENGTH = 2 +# If source is short (under this many chars), skip the ratio check entirely +# — short strings are too noisy to be useful for length analysis. +MIN_SOURCE_LENGTH_FOR_RATIO = 20 + +# Pattern to detect text that is mostly digits / numbers / simple symbols. +# E.g. "Price: 100€", "+33 6 12 34 56 78", "192.168.1.1". +_MOSTLY_NUMERIC_RE = re.compile(r"^[\d\s\W]*$", re.UNICODE) +_NUMERIC_RATIO_THRESHOLD = 0.5 # 50% of letters are digits + + +def check(source_text: str, translated_text: str) -> Dict: + """ + Returns a dict like: + { + "issue": None | "length_outlier" | "truncation_suspect", + "ratio": float, + "source_length": int, + "translated_length": int, + } + Never raises. + """ + if not source_text: + return { + "issue": None, + "ratio": None, + "source_length": 0, + "translated_length": len(translated_text or ""), + } + + src_len = len(source_text.strip()) + trans_len = len(translated_text.strip()) + + # Empty translation — always suspect. + if trans_len == 0: + return { + "issue": "truncation_suspect", + "ratio": 0.0, + "source_length": src_len, + "translated_length": trans_len, + } + + # If source is mostly digits/numbers, the translation can also be short + # (e.g. "Price: 100" → "100"). Don't flag length in this case. + if _is_mostly_numeric(source_text): + return { + "issue": None, + "ratio": None, + "source_length": src_len, + "translated_length": trans_len, + "note": "skipped: numeric source", + } + + # If source is very short, skip the ratio check. + if src_len < MIN_SOURCE_LENGTH_FOR_RATIO: + return { + "issue": None, + "ratio": None, + "source_length": src_len, + "translated_length": trans_len, + } + + ratio = trans_len / src_len + + if ratio > RATIO_MAX: + return { + "issue": "length_outlier", + "ratio": round(ratio, 2), + "source_length": src_len, + "translated_length": trans_len, + } + if ratio < RATIO_MIN: + return { + "issue": "truncation_suspect", + "ratio": round(ratio, 2), + "source_length": src_len, + "translated_length": trans_len, + } + + return { + "issue": None, + "ratio": round(ratio, 2), + "source_length": src_len, + "translated_length": trans_len, + } + + +def _is_mostly_numeric(text: str) -> bool: + """True if at least 50% of non-whitespace characters are digits.""" + if not text: + return False + chars = [c for c in text if not c.isspace()] + if not chars: + return False + digit_count = sum(1 for c in chars if c.isdigit()) + return (digit_count / len(chars)) >= _NUMERIC_RATIO_THRESHOLD diff --git a/services/quality/pattern_leak.py b/services/quality/pattern_leak.py new file mode 100644 index 0000000..d24990d --- /dev/null +++ b/services/quality/pattern_leak.py @@ -0,0 +1,126 @@ +""" +Pattern leak detection for the L0 quality layer. + +Detects common failure modes where the LLM translator: + 1. Leaks parts of the system prompt (e.g. starts with "Translation:" or "Voici la traduction :") + 2. Hallucinates by repeating the same word/phrase many times in a row + 3. Returns a chain-of-thought / explanation instead of a translation + +These checks are pure regex / counting — no model, no network. +""" + +from __future__ import annotations + +import re +from typing import Dict, List + + +# ---------- Prompt leak patterns ---------- + +# Phrases that strongly suggest the LLM leaked its prompt or a thought process +# into the output. We check only at the START of the translation (after +# stripping whitespace) to avoid false positives on legitimate text. +LEAK_PREFIX_PATTERNS: List[re.Pattern] = [ + re.compile(r"^(translation|translated text|here is the translation|here'?s the translation)\s*[:：-]", re.IGNORECASE), + re.compile(r"^(voici (la |ma )?traduction|traduction\s*[:：-])\b", re.IGNORECASE), + re.compile(r"^(原文|译|翻译|译为|以下是)\s*[:：]?", re.UNICODE), + re.compile(r"^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)", re.IGNORECASE), + re.compile(r"^(\*\*|__|\#)\s*translation", re.IGNORECASE), + re.compile(r"^translated from\s+\w+\s+to\s+\w+\s*[:：-]", re.IGNORECASE), +] + + +# ---------- Repetition detection ---------- + +# A "word" for the repetition check — uses Unicode word boundaries so +# it works on Chinese, Japanese, Korean, etc. +_WORD_RE = re.compile(r"\S+", re.UNICODE) + +# Threshold: a word (or token) repeated 5+ times consecutively is almost +# always a hallucination. We allow up to 4 to handle legitimate text +# like "the the" (typo) without false positives. +REPETITION_THRESHOLD = 5 +# Same character repeated 20+ times in a row is also a hallucination +# (catches cases like "xxxxxxxxxxx" or "=========="). +CHAR_REPETITION_THRESHOLD = 20 + + +def check(text: str) -> Dict: + """ + Returns a dict like: + { + "issue": None | "prompt_leak" | "repetition_hallucination", + "matched_pattern": "..." | None, + "repetition_count": int | None, + } + Never raises. + """ + if not text or not text.strip(): + return {"issue": None, "matched_pattern": None, "repetition_count": None} + + stripped = text.lstrip() + + # 1. Prompt leak + for pat in LEAK_PREFIX_PATTERNS: + m = pat.match(stripped) + if m: + return { + "issue": "prompt_leak", + "matched_pattern": pat.pattern, + "repetition_count": None, + } + + # 2. Token-level repetition + tokens = _WORD_RE.findall(stripped) + rep_count = _max_consecutive_repetition(tokens) + if rep_count >= REPETITION_THRESHOLD: + return { + "issue": "repetition_hallucination", + "matched_pattern": None, + "repetition_count": rep_count, + } + + # 3. Character-level repetition (catches "xxxxxx" without spaces) + char_rep = _max_consecutive_char_repetition(stripped) + if char_rep >= CHAR_REPETITION_THRESHOLD: + return { + "issue": "repetition_hallucination", + "matched_pattern": None, + "repetition_count": char_rep, + } + + return {"issue": None, "matched_pattern": None, "repetition_count": max(rep_count, char_rep) or None} + + +def _max_consecutive_repetition(tokens: List[str]) -> int: + """Return the maximum number of times the same token appears consecutively.""" + if not tokens: + return 0 + # Normalize: lower-case + strip basic punctuation for comparison + norm = [t.lower().strip(".,!?;:\"'`()[]{}") for t in tokens] + max_run = 1 + current_run = 1 + for i in range(1, len(norm)): + if norm[i] and norm[i] == norm[i - 1]: + current_run += 1 + if current_run > max_run: + max_run = current_run + else: + current_run = 1 + return max_run + + +def _max_consecutive_char_repetition(text: str) -> int: + """Return the maximum number of times the same character appears consecutively.""" + if not text: + return 0 + max_run = 1 + current_run = 1 + for i in range(1, len(text)): + if text[i] == text[i - 1] and not text[i].isspace(): + current_run += 1 + if current_run > max_run: + max_run = current_run + else: + current_run = 1 + return max_run diff --git a/services/quality/pipeline.py b/services/quality/pipeline.py new file mode 100644 index 0000000..e9bb6f6 --- /dev/null +++ b/services/quality/pipeline.py @@ -0,0 +1,75 @@ +""" +Quality pipeline — defensive wrapper around the L0 checks. + +The pipeline is the integration point for the route. It: + 1. Catches all exceptions (L0 must NEVER break a translation job) + 2. Adds timing + 3. Emits a single structured log line per job + +The actual checks live in `script_detector`, `length_checker`, `pattern_leak`. +This module is the orchestration / safety layer. +""" + +from __future__ import annotations + +import time +from typing import List, Optional + +from core.logging import get_logger + +from .script_detector import evaluate_document, DocumentQualityResult + +logger = get_logger(__name__) + + +def run_l0_check( + source_chunks: List[str], + translated_chunks: List[str], + target_lang: Optional[str], + job_id: Optional[str] = None, + file_extension: Optional[str] = None, +) -> DocumentQualityResult: + """ + Run the L0 quality checks defensively. Never raises. + + Returns an empty/neutral DocumentQualityResult on internal error + so the calling route can log and continue without affecting the + translation job outcome. + """ + start = time.time() + empty = DocumentQualityResult( + passed=True, + score=0.0, + chunk_count=0, + failed_chunk_count=0, + issues={"internal_error": 1}, + ) + + try: + result = evaluate_document(source_chunks, translated_chunks, target_lang) + except Exception as e: + elapsed_ms = round((time.time() - start) * 1000, 2) + logger.warning( + "quality_l0_check_failed", + job_id=job_id, + file_extension=file_extension, + error=str(e)[:200], + error_type=type(e).__name__, + elapsed_ms=elapsed_ms, + ) + return empty + + elapsed_ms = round((time.time() - start) * 1000, 2) + logger.info( + "quality_l0_check", + job_id=job_id, + file_extension=file_extension, + target_lang=target_lang, + chunk_count=result.chunk_count, + failed_chunk_count=result.failed_chunk_count, + score=result.score, + passed=result.passed, + issues=result.issues, + elapsed_ms=elapsed_ms, + ) + return result diff --git a/services/quality/script_detector.py b/services/quality/script_detector.py new file mode 100644 index 0000000..21f5fd5 --- /dev/null +++ b/services/quality/script_detector.py @@ -0,0 +1,374 @@ +""" +L0 script detector. + +Verifies that a translated string is actually written in the script expected +for the target language. + +This is the first line of defense against the most common translation +failure mode: the LLM hallucinates text in the wrong language or wrong +script (e.g. user asks for Persian, model returns Arabic, or user asks +for Hindi, model returns Arabic). The check is purely heuristic — it +counts code points in the relevant Unicode ranges and compares to a +threshold. + +Pure Python. No network calls. No new dependencies. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from typing import Dict, List, Optional + +from . import config as _config +from . import length_checker +from . import pattern_leak + +from core.logging import get_logger + +logger = get_logger(__name__) + + +# ---------- Result dataclasses ---------- + +@dataclass +class QualityCheckResult: + """Result of evaluating a single (source, translation) pair.""" + passed: bool + score: float # 0.0 to 1.0 + issues: List[str] = field(default_factory=list) + detected_script: Optional[str] = None + expected_script: Optional[str] = None + details: Dict = field(default_factory=dict) + + def to_log_dict(self) -> Dict: + return asdict(self) + + +@dataclass +class DocumentQualityResult: + """Aggregated result for a list of (source, translation) pairs.""" + passed: bool + score: float # mean score across chunks + chunk_count: int + failed_chunk_count: int + issues: Dict[str, int] = field(default_factory=dict) # issue -> count + samples: List[Dict] = field(default_factory=list) # a few example failures + + def to_log_dict(self) -> Dict: + return asdict(self) + + +# ---------- Core helpers ---------- + +def _char_in_ranges(code_point: int, ranges: list) -> bool: + """True if a code point falls in any of the (start, end) ranges.""" + for start, end in ranges: + if start <= code_point <= end: + return True + return False + + +def _count_letters(text: str) -> int: + """Count alphabetic characters (using Python's built-in isalpha).""" + return sum(1 for c in text if c.isalpha()) + + +def _count_in_script(text: str, ranges: list) -> int: + """Count how many alphabetic characters fall within the given Unicode ranges.""" + if not ranges: + # 'latin' or unknown — treat all letters as matching. + return _count_letters(text) + return sum( + 1 for c in text + if c.isalpha() and _char_in_ranges(ord(c), ranges) + ) + + +# ---------- Arabic-script variant detection ---------- + +def detect_arabic_variant( + text: str, + claimed_lang: Optional[str], +) -> Dict: + """ + For text that is in the Arabic script block, check whether it matches + the specific variant the user asked for (Persian, Urdu, Pashto, etc.). + + Returns a dict like: + { + "verdict": "pass" | "fail" | "skip", + "claimed_lang": "fa", + "detected_variants": ["fa"], + "reason": "...", + } + + Detection logic: + 1. If the text has < 60% Arabic-script letters overall, verdict = "skip" + (the script-detector will catch the mismatch). + 2. If claimed_lang is NOT an Arabic-script language, verdict = "fail" + (this case should have been caught upstream — defensive double-check). + 3. Scan the text for any discriminating character from any + Arabic-script language. If a discriminating character of a + DIFFERENT language is found, verdict = "fail". + 4. Otherwise verdict = "pass". + """ + if not text or not text.strip(): + return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "empty text"} + + arabic_ranges = _config.get_ranges("arabic") + letters = _count_letters(text) + if letters == 0: + return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "no letters"} + + in_arabic = _count_in_script(text, arabic_ranges) + arabic_ratio = in_arabic / letters + + if arabic_ratio < _config.MIN_RATIO_IN_SCRIPT: + # Not really Arabic-script — let the main script_detector handle it. + return { + "verdict": "skip", + "claimed_lang": claimed_lang, + "arabic_ratio": round(arabic_ratio, 3), + "reason": "not in Arabic script", + } + + if not _config.is_arabic_script_lang(claimed_lang): + # The translation IS in Arabic but the target wasn't Arabic. + # The main script_detector will fail on this; we just return skip. + return { + "verdict": "skip", + "claimed_lang": claimed_lang, + "arabic_ratio": round(arabic_ratio, 3), + "reason": "target is not an Arabic-script language", + } + + # Now: text is Arabic-script AND target is Arabic-script. Check the variant. + detected = set() + for lang_code, chars in _config.DISCRIMINATING_CHARS.items(): + if not chars: + continue + if any(c in chars for c in text): + detected.add(lang_code) + + if detected and claimed_lang and claimed_lang.lower() not in detected: + return { + "verdict": "fail", + "claimed_lang": claimed_lang, + "detected_variants": sorted(detected), + "arabic_ratio": round(arabic_ratio, 3), + "reason": ( + f"target={claimed_lang} but text contains characters typical of " + f"{', '.join(sorted(detected))}" + ), + } + + return { + "verdict": "pass", + "claimed_lang": claimed_lang, + "detected_variants": sorted(detected) if detected else [claimed_lang], + "arabic_ratio": round(arabic_ratio, 3), + "reason": "ok", + } + + +# ---------- Per-chunk evaluation ---------- + +def evaluate_chunk( + source_text: str, + translated_text: str, + target_lang: Optional[str], +) -> QualityCheckResult: + """ + Run the L0 checks on a single (source, translation) pair. + + Returns a QualityCheckResult. The function is purely defensive — it + never raises; any internal error results in a "skip" result. + """ + if translated_text is None: + return QualityCheckResult( + passed=True, score=0.0, issues=["empty_translation"], + details={"reason": "translation is None"}, + ) + + text = translated_text.strip() + if not text: + return QualityCheckResult( + passed=True, score=0.0, issues=["empty_translation"], + details={"reason": "translation is empty or whitespace-only"}, + ) + + target_lang = (target_lang or "").lower() or None + issues: List[str] = [] + details: Dict = {} + + # --- Script detection --- + expected_script = _config.get_script(target_lang) + expected_ranges = _config.get_ranges(expected_script) + letters = _count_letters(text) + + if letters == 0: + # No alphabetic characters — could be numbers, punctuation, or + # a single non-Latin symbol. Skip script check. + script_score = 1.0 + detected_script = expected_script + details["script_check"] = "skipped: no alphabetic characters" + else: + # Always try to determine the ACTUAL script of the text — used for + # diagnostics and for catching language confusion when the target + # is Latin (e.g. user asks fr, we get Arabic text). + detected_script = _detect_actual_script(text) + in_expected = _count_in_script(text, expected_ranges) + script_score = in_expected / letters + + details["script_score"] = round(script_score, 3) + details["letters_in_text"] = letters + details["letters_in_script"] = in_expected + details["detected_script"] = detected_script + details["expected_script"] = expected_script + details["min_ratio"] = _config.MIN_RATIO_IN_SCRIPT + + # Two failure modes: + # 1. Target is a SPECIFIC non-Latin script (cyrillic, arabic, cjk...) + # and the text doesn't match it enough. + # 2. Target is Latin but the text is clearly in a SPECIFIC other + # script (cyrillic, arabic, devanagari, cjk...) — language + # confusion. + if expected_script != "latin" and expected_ranges: + # Specific non-Latin target. + if script_score < _config.MIN_RATIO_IN_SCRIPT: + issues.append("wrong_script") + details["reason"] = ( + f"only {script_score:.0%} of letters match {expected_script} script; " + f"text appears to be in {detected_script}" + ) + else: + # Latin target. If detected script is clearly non-Latin, fail. + if detected_script and detected_script != "latin" and detected_script != "unknown": + # Measure how confident we are that the text is non-Latin. + non_latin_ranges = _config.get_ranges(detected_script) + in_detected = _count_in_script(text, non_latin_ranges) + non_latin_confidence = in_detected / letters + if non_latin_confidence >= 0.7: + issues.append("wrong_script") + details["reason"] = ( + f"target is Latin but {non_latin_confidence:.0%} of letters " + f"are in {detected_script} script — language confusion" + ) + + # --- Arabic-script variant detection --- + if _config.is_arabic_script_lang(target_lang): + variant_result = detect_arabic_variant(text, target_lang) + details["arabic_variant"] = variant_result + if variant_result["verdict"] == "fail": + issues.append("wrong_arabic_variant") + + # --- Length sanity --- + length_result = length_checker.check(source_text, text) + details["length"] = length_result + if length_result.get("issue"): + issues.append(length_result["issue"]) + + # --- Pattern leak / repetition --- + leak_result = pattern_leak.check(text) + details["pattern_check"] = leak_result + if leak_result.get("issue"): + issues.append(leak_result["issue"]) + + # --- Aggregate --- + passed = len(issues) == 0 + # Simple score: how many of the 3 main checks passed. + n_checks = 3 + n_failed = sum( + 1 for issue in issues if issue in ( + "wrong_script", "wrong_arabic_variant", + "length_outlier", "truncation_suspect", + "prompt_leak", "repetition_hallucination", + ) + ) + score = max(0.0, 1.0 - (n_failed / n_checks)) + + return QualityCheckResult( + passed=passed, + score=round(score, 3), + issues=issues, + detected_script=detected_script, + expected_script=expected_script, + details=details, + ) + + +def _detect_actual_script(text: str) -> str: + """ + Heuristically determine which script a string is in. Used only for + diagnostics — never for the verdict. Returns the first script (in + priority order) whose ratio exceeds the threshold. + """ + letters = _count_letters(text) + if letters == 0: + return "unknown" + # Priority order: more specific scripts first. + order = [ + "hiragana_katakana", "hangul", "cjk", "thai", "lao", "burmese", + "khmer", "devanagari", "bengali", "tamil", "telugu", "kannada", + "malayalam", "sinhala", "gujarati", "gurmukhi", + "arabic", "hebrew", "cyrillic", "greek", "armenian", "georgian", + "ethiopic", "tibetan", "thaana", + ] + for script_id in order: + ranges = _config.get_ranges(script_id) + in_script = _count_in_script(text, ranges) + if in_script / letters > 0.4: + return script_id + return "latin" + + +# ---------- Document-level aggregation ---------- + +def evaluate_document( + source_chunks: List[str], + translated_chunks: List[str], + target_lang: Optional[str], + sample_size: int = 50, +) -> DocumentQualityResult: + """ + Evaluate all (source, translation) pairs and return a document-level + summary. The full list is processed but only the first `sample_size` + failing chunks are kept in `samples` to keep logs compact. + """ + n = min(len(source_chunks), len(translated_chunks)) + chunk_results: List[QualityCheckResult] = [] + issues_count: Dict[str, int] = {} + samples: List[Dict] = [] + score_sum = 0.0 + failed_count = 0 + + for i in range(n): + r = evaluate_chunk(source_chunks[i], translated_chunks[i], target_lang) + chunk_results.append(r) + score_sum += r.score + for issue in r.issues: + issues_count[issue] = issues_count.get(issue, 0) + 1 + if not r.passed: + failed_count += 1 + if len(samples) < sample_size: + src_preview = (source_chunks[i] or "")[:80] + trans_preview = (translated_chunks[i] or "")[:80] + samples.append({ + "index": i, + "issues": r.issues, + "source_preview": src_preview, + "translated_preview": trans_preview, + "details": r.details, + }) + + mean_score = (score_sum / n) if n > 0 else 0.0 + passed = failed_count == 0 + + return DocumentQualityResult( + passed=passed, + score=round(mean_score, 3), + chunk_count=n, + failed_chunk_count=failed_count, + issues=issues_count, + samples=samples, + ) diff --git a/tests/services/quality/__init__.py b/tests/services/quality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/quality/test_config.py b/tests/services/quality/test_config.py new file mode 100644 index 0000000..d9ce467 --- /dev/null +++ b/tests/services/quality/test_config.py @@ -0,0 +1,143 @@ +""" +Tests for services/quality/config.py +Covers the language → script mapping and discriminating characters. +""" +import pytest + +from services.quality import config as qconfig + + +class TestGetScript: + def test_cyrillic_languages(self): + assert qconfig.get_script("ru") == "cyrillic" + assert qconfig.get_script("uk") == "cyrillic" + assert qconfig.get_script("be") == "cyrillic" + assert qconfig.get_script("bg") == "cyrillic" + assert qconfig.get_script("sr") == "cyrillic" + assert qconfig.get_script("kk") == "cyrillic" + + def test_latin_languages(self): + assert qconfig.get_script("en") == "latin" + assert qconfig.get_script("fr") == "latin" + assert qconfig.get_script("de") == "latin" + assert qconfig.get_script("es") == "latin" + assert qconfig.get_script("vi") == "latin" + assert qconfig.get_script("tr") == "latin" + + def test_cjk_languages(self): + assert qconfig.get_script("zh") == "cjk" + assert qconfig.get_script("zh-cn") == "cjk" + assert qconfig.get_script("zh-tw") == "cjk" + + def test_japanese_uses_kana(self): + # ja is mapped to hiragana_katakana specifically so it can be + # distinguished from Chinese CJK. + assert qconfig.get_script("ja") == "hiragana_katakana" + + def test_korean_uses_hangul(self): + assert qconfig.get_script("ko") == "hangul" + + def test_arabic_script_languages(self): + assert qconfig.get_script("ar") == "arabic" + assert qconfig.get_script("fa") == "arabic" + assert qconfig.get_script("ur") == "arabic" + assert qconfig.get_script("ps") == "arabic" + assert qconfig.get_script("ku") == "arabic" + + def test_hebrew_and_yiddish(self): + assert qconfig.get_script("he") == "hebrew" + # Yiddish uses Hebrew script, not Arabic + assert qconfig.get_script("yi") == "hebrew" + + def test_thaana(self): + # Maldivian (Dhivehi) uses Thaana script, not Arabic + assert qconfig.get_script("dv") == "thaana" + + def test_indian_scripts(self): + assert qconfig.get_script("hi") == "devanagari" + assert qconfig.get_script("bn") == "bengali" + assert qconfig.get_script("ta") == "tamil" + assert qconfig.get_script("te") == "telugu" + assert qconfig.get_script("gu") == "gujarati" + assert qconfig.get_script("pa") == "gurmukhi" + assert qconfig.get_script("ml") == "malayalam" + assert qconfig.get_script("kn") == "kannada" + assert qconfig.get_script("si") == "sinhala" + + def test_se_asian_scripts(self): + assert qconfig.get_script("th") == "thai" + assert qconfig.get_script("lo") == "lao" + assert qconfig.get_script("my") == "burmese" + assert qconfig.get_script("km") == "khmer" + + def test_other_scripts(self): + assert qconfig.get_script("el") == "greek" + assert qconfig.get_script("ka") == "georgian" + assert qconfig.get_script("hy") == "armenian" + assert qconfig.get_script("am") == "ethiopic" + assert qconfig.get_script("bo") == "tibetan" + + def test_unknown_falls_back_to_latin(self): + assert qconfig.get_script("xx") == "latin" + assert qconfig.get_script("") == "latin" + assert qconfig.get_script(None) == "latin" + + def test_case_insensitive(self): + assert qconfig.get_script("FR") == "latin" + assert qconfig.get_script("ZH-CN") == "cjk" + assert qconfig.get_script("Ru") == "cyrillic" + + +class TestIsArabicScriptLang: + def test_true_for_arabic_script_langs(self): + assert qconfig.is_arabic_script_lang("ar") is True + assert qconfig.is_arabic_script_lang("fa") is True + assert qconfig.is_arabic_script_lang("ur") is True + assert qconfig.is_arabic_script_lang("ckb") is True + + def test_false_for_non_arabic(self): + assert qconfig.is_arabic_script_lang("fr") is False + assert qconfig.is_arabic_script_lang("he") is False + assert qconfig.is_arabic_script_lang("hi") is False + assert qconfig.is_arabic_script_lang("en") is False + + def test_false_for_unknown(self): + assert qconfig.is_arabic_script_lang("xx") is False + assert qconfig.is_arabic_script_lang("") is False + assert qconfig.is_arabic_script_lang(None) is False + + +class TestDiscriminatingChars: + def test_persian_chars(self): + chars = qconfig.get_discriminating_chars("fa") + assert "پ" in chars # peh + assert "چ" in chars # tcheh + assert "ژ" in chars # zheh + assert "گ" in chars # guaf + + def test_urdu_chars(self): + chars = qconfig.get_discriminating_chars("ur") + assert "ٹ" in chars # tteh + assert "ڈ" in chars # ddal + + def test_unknown_lang_empty(self): + assert qconfig.get_discriminating_chars("fr") == frozenset() + assert qconfig.get_discriminating_chars("") == frozenset() + assert qconfig.get_discriminating_chars(None) == frozenset() + + +class TestGetRanges: + def test_latin_returns_empty(self): + # Latin is the fallback — no ranges, means "anything matches". + assert qconfig.get_ranges("latin") == [] + + def test_cyrillic_ranges(self): + ranges = qconfig.get_ranges("cyrillic") + assert any(start == 0x0400 for start, _ in ranges) + + def test_cjk_ranges(self): + ranges = qconfig.get_ranges("cjk") + assert any(start == 0x4E00 for start, _ in ranges) + + def test_unknown_script_returns_empty(self): + assert qconfig.get_ranges("mystery_script") == [] diff --git a/tests/services/quality/test_file_extractor.py b/tests/services/quality/test_file_extractor.py new file mode 100644 index 0000000..9faf0cf --- /dev/null +++ b/tests/services/quality/test_file_extractor.py @@ -0,0 +1,181 @@ +""" +Tests for services/quality/file_extractor.py +Uses real files generated via temporary paths. + +On Windows, tempfile.NamedTemporaryFile holds the file open and blocks +overwrite, so we use a plain temp directory + manual filename instead. +""" +import tempfile +import os +from pathlib import Path + +import pytest + +from services.quality.file_extractor import ( + extract_sample, + DEFAULT_MAX_SAMPLES, +) + + +class TestExtractSample: + def test_returns_empty_for_missing_file(self): + result = extract_sample(Path("/nonexistent/path/file.docx"), ".docx") + assert result == [] + + def test_returns_empty_for_none_path(self): + result = extract_sample(None, ".docx") + assert result == [] + + def test_returns_empty_for_unsupported_extension(self): + # .txt isn't supported — should return [] silently + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "test.txt" + p.write_text("some text") + result = extract_sample(p, ".txt") + assert result == [] + + +def _make_tmp_path(suffix: str) -> Path: + """Create a unique temp file path. File does not exist yet.""" + fd, name = tempfile.mkstemp(suffix=suffix) + os.close(fd) + return Path(name) + + +class TestDocxExtraction: + def test_extracts_paragraphs(self): + from docx import Document + doc = Document() + doc.add_paragraph("Bonjour le monde") + doc.add_paragraph("Comment allez-vous?") + doc.add_paragraph("Merci beaucoup") + p = _make_tmp_path(".docx") + try: + doc.save(str(p)) + result = extract_sample(p, ".docx", max_samples=10) + assert len(result) >= 1 + texts = [s["translated"] for s in result] + assert "Bonjour le monde" in texts + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + def test_respects_max_samples(self): + from docx import Document + doc = Document() + for i in range(50): + doc.add_paragraph(f"Paragraphe numero {i} avec du texte") + p = _make_tmp_path(".docx") + try: + doc.save(str(p)) + result = extract_sample(p, ".docx", max_samples=5) + assert len(result) == 5 + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + def test_handles_empty_doc(self): + from docx import Document + doc = Document() + p = _make_tmp_path(".docx") + try: + doc.save(str(p)) + result = extract_sample(p, ".docx") + assert result == [] + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + +class TestXlsxExtraction: + def test_extracts_cells(self): + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws["A1"] = "Bonjour" + ws["A2"] = "Monde" + ws["A3"] = "Comment" + p = _make_tmp_path(".xlsx") + try: + wb.save(str(p)) + wb.close() + result = extract_sample(p, ".xlsx", max_samples=10) + assert len(result) >= 1 + texts = [s["translated"] for s in result] + assert "Bonjour" in texts + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + def test_skips_numeric_cells(self): + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws["A1"] = 100 + ws["A2"] = 200 + ws["A3"] = "Hello" + p = _make_tmp_path(".xlsx") + try: + wb.save(str(p)) + wb.close() + result = extract_sample(p, ".xlsx") + texts = [s["translated"] for s in result] + assert "Hello" in texts + assert 100 not in texts # numeric only cells skipped + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + +class TestPptxExtraction: + def test_extracts_slide_text(self): + from pptx import Presentation + pres = Presentation() + slide = pres.slides.add_slide(pres.slide_layouts[0]) + slide.shapes.title.text = "Bonjour le monde" + p = _make_tmp_path(".pptx") + try: + pres.save(str(p)) + result = extract_sample(p, ".pptx") + assert len(result) >= 1 + assert result[0]["translated"] == "Bonjour le monde" + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass + + +class TestPdfExtraction: + def test_extracts_pdf_text(self): + # Create a minimal PDF using reportlab if available, else skip + pytest.importorskip("fitz") + try: + from reportlab.pdfgen import canvas + except ImportError: + pytest.skip("reportlab not available") + p = _make_tmp_path(".pdf") + try: + c = canvas.Canvas(str(p)) + c.drawString(100, 750, "Bonjour le monde") + c.drawString(100, 700, "Comment allez vous") + c.save() + result = extract_sample(p, ".pdf") + assert len(result) >= 1 + texts = [s["translated"] for s in result] + assert any("Bonjour" in t for t in texts) + finally: + try: + p.unlink() + except (OSError, PermissionError): + pass diff --git a/tests/services/quality/test_length_checker.py b/tests/services/quality/test_length_checker.py new file mode 100644 index 0000000..bfe605b --- /dev/null +++ b/tests/services/quality/test_length_checker.py @@ -0,0 +1,49 @@ +""" +Tests for services/quality/length_checker.py +""" +import pytest + +from services.quality.length_checker import check + + +class TestLengthCheck: + def test_normal_length(self): + r = check("Hello world, this is a test of translation length", + "Bonjour le monde, ceci est un test de longueur de traduction") + assert r["issue"] is None + assert r["ratio"] is not None + assert 0.5 < r["ratio"] < 2.0 + + def test_huge_translation(self): + src = "A" * 200 + r = check(src, "x" * 1000) + assert r["issue"] == "length_outlier" + assert r["ratio"] > 3.5 + + def test_tiny_translation(self): + r = check("A" * 100, "ok") + assert r["issue"] == "truncation_suspect" + assert r["ratio"] < 0.15 + + def test_empty_translation_flagged(self): + r = check("Hello world, this is a test", "") + assert r["issue"] == "truncation_suspect" + + def test_short_source_skips_ratio(self): + r = check("Hi", "ok") + assert r["issue"] is None + assert r["ratio"] is None + + def test_empty_inputs(self): + r = check("", "") + assert r["issue"] is None + assert r["source_length"] == 0 + assert r["translated_length"] == 0 + + def test_none_source(self): + r = check(None, "translation") + assert r["issue"] is None + + def test_numeric_source(self): + r = check("Price: 100", "100") + assert r["issue"] is None # numeric sources skip the check diff --git a/tests/services/quality/test_pattern_leak.py b/tests/services/quality/test_pattern_leak.py new file mode 100644 index 0000000..2448a02 --- /dev/null +++ b/tests/services/quality/test_pattern_leak.py @@ -0,0 +1,85 @@ +""" +Tests for services/quality/pattern_leak.py +""" +import pytest + +from services.quality.pattern_leak import check + + +class TestPromptLeak: + def test_english_translation_prefix(self): + r = check("Translation: Bonjour le monde") + assert r["issue"] == "prompt_leak" + + def test_here_is_translation(self): + r = check("Here is the translation: Bonjour") + assert r["issue"] == "prompt_leak" + + def test_french_voici_traduction(self): + r = check("Voici la traduction : Bonjour le monde") + assert r["issue"] == "prompt_leak" + + def test_chinese_translation_prefix(self): + r = check("翻译：你好世界") + assert r["issue"] == "prompt_leak" + + def test_sure_heres_translation(self): + r = check("Sure, here's the translation: Hello world") + assert r["issue"] == "prompt_leak" + + def test_of_course_heres(self): + r = check("Of course, here you go: Bonjour") + assert r["issue"] == "prompt_leak" + + def test_markdown_bold_translation(self): + r = check("**Translation** Bonjour le monde") + assert r["issue"] == "prompt_leak" + + def test_normal_text_passes(self): + r = check("Bonjour le monde, comment allez-vous?") + assert r["issue"] is None + + def test_text_with_translation_word_in_middle_passes(self): + r = check("Voici ce que je pense de la traduction de ce texte") + # The word "traduction" appears but not as a prefix + assert r["issue"] is None + + +class TestRepetitionHallucination: + def test_long_repetition_detected(self): + r = check("the the the the the the the the") + assert r["issue"] == "repetition_hallucination" + assert r["repetition_count"] >= 5 + + def test_short_repetition_passes(self): + # 4 times is within tolerance + r = check("I think I think I think I think") + assert r["issue"] is None + + def test_normal_text_passes(self): + r = check("This is a normal sentence with no repetition issues") + assert r["issue"] is None + + def test_mixed_repetition_in_long_text_passes(self): + # Repetition is local, not a hallucination + r = check("The cat sat on the mat. The cat was happy. The dog was sad.") + assert r["issue"] is None + + def test_repetition_with_punctuation(self): + # "the, the, the, the, the" should still be detected + r = check("the, the, the, the, the, the") + assert r["issue"] == "repetition_hallucination" + + +class TestEmptyAndEdgeCases: + def test_empty_text(self): + r = check("") + assert r["issue"] is None + + def test_whitespace_only(self): + r = check(" \n \t ") + assert r["issue"] is None + + def test_none_input(self): + r = check(None) + assert r["issue"] is None diff --git a/tests/services/quality/test_pipeline.py b/tests/services/quality/test_pipeline.py new file mode 100644 index 0000000..fda977a --- /dev/null +++ b/tests/services/quality/test_pipeline.py @@ -0,0 +1,61 @@ +""" +Tests for services/quality/pipeline.py +""" +import pytest + +from services.quality import run_l0_check +from services.quality.script_detector import DocumentQualityResult + + +class TestRunL0Check: + def test_returns_result_on_success(self): + result = run_l0_check( + ["Hello", "World"], + ["Bonjour", "Monde"], + "fr", + job_id="test_job_1", + ) + assert isinstance(result, DocumentQualityResult) + assert result.passed is True + assert result.chunk_count == 2 + + def test_returns_neutral_on_empty_lists(self): + result = run_l0_check([], [], "fr", job_id="test_job_2") + assert result.passed is True + assert result.chunk_count == 0 + + def test_detects_failures(self): + result = run_l0_check( + ["Hello", "World"], + ["Bonjour", "مرحبا"], + "fr", + job_id="test_job_3", + ) + assert result.passed is False + assert "wrong_script" in result.issues + + def test_never_raises_on_bad_input(self): + # Even with weird input, it shouldn't raise + result = run_l0_check( + ["Hello"], + [None], # None translation + "fr", + job_id="test_job_4", + ) + assert result is not None + + def test_optional_job_id(self): + # job_id is optional + result = run_l0_check(["hi"], ["salut"], "fr") + assert result is not None + + def test_optional_file_extension(self): + result = run_l0_check( + ["hi"], ["salut"], "fr", file_extension=".docx" + ) + assert result is not None + + def test_target_lang_none(self): + # Should not crash with no target lang + result = run_l0_check(["hi"], ["salut"], None) + assert result is not None diff --git a/tests/services/quality/test_script_detector.py b/tests/services/quality/test_script_detector.py new file mode 100644 index 0000000..61e89bd --- /dev/null +++ b/tests/services/quality/test_script_detector.py @@ -0,0 +1,290 @@ +""" +Tests for services/quality/script_detector.py +""" +import pytest + +from services.quality import evaluate_chunk, evaluate_document, detect_arabic_variant + + +# ---------- Happy path: correct script ---------- + +class TestCorrectScript: + def test_russian_correct(self): + r = evaluate_chunk("Hello world", "Привет мир", "ru") + assert r.passed is True + assert "wrong_script" not in r.issues + + def test_chinese_simplified_correct(self): + r = evaluate_chunk("Hello", "你好世界", "zh") + assert r.passed is True + + def test_arabic_correct(self): + r = evaluate_chunk("Hello", "مرحبا بالعالم", "ar") + assert r.passed is True + + def test_persian_correct(self): + # Persian has unique chars چ پ ژ گ + r = evaluate_chunk("Hello", "سلام چطوری؟ من پژوهشگر هستم", "fa") + assert r.passed is True + + def test_french_correct(self): + r = evaluate_chunk("Hello world", "Bonjour le monde", "fr") + assert r.passed is True + + def test_hebrew_correct(self): + r = evaluate_chunk("Hello", "שלום עולם", "he") + assert r.passed is True + + def test_korean_correct(self): + r = evaluate_chunk("Hello", "안녕하세요 세계", "ko") + assert r.passed is True + + def test_japanese_correct(self): + # Japanese must contain hiragana or katakana to be classified as ja. + r = evaluate_chunk("Hello", "こんにちは世界", "ja") + assert r.passed is True + + def test_hindi_correct(self): + r = evaluate_chunk("Hello", "नमस्ते दुनिया", "hi") + assert r.passed is True + + def test_thai_correct(self): + r = evaluate_chunk("Hello", "สวัสดีชาวโลก", "th") + assert r.passed is True + + def test_greek_correct(self): + r = evaluate_chunk("Hello", "Γεια σας κόσμε", "el") + assert r.passed is True + + +# ---------- Wrong script: language confusion ---------- + +class TestWrongScript: + def test_french_text_for_japanese_target(self): + r = evaluate_chunk("Hello", "Bonjour le monde", "ja") + assert r.passed is False + assert "wrong_script" in r.issues + + def test_arabic_text_for_french_target(self): + r = evaluate_chunk("Hello", "مرحبا بالعالم", "fr") + assert r.passed is False + assert "wrong_script" in r.issues + + def test_russian_text_for_english_target(self): + r = evaluate_chunk("Hello", "Привет мир", "en") + assert r.passed is False + assert "wrong_script" in r.issues + + def test_chinese_text_for_korean_target(self): + r = evaluate_chunk("Hello", "你好世界", "ko") + # Hangul syllables are 0xAC00-0xD7AF — Chinese chars are not in that range. + assert r.passed is False + assert "wrong_script" in r.issues + + +# ---------- Persian / Arabic discrimination ---------- + +class TestArabicVariantDiscrimination: + def test_persian_for_arabic_target_fails(self): + # Persian text with چ پ ژ گ should fail when target is Arabic. + r = evaluate_chunk("Hello", "سلام چطوری؟", "ar") + assert r.passed is False + assert "wrong_arabic_variant" in r.issues + + def test_arabic_for_persian_target_fails(self): + # Pure Arabic text (no Persian-specific chars) when target is Persian. + r = evaluate_chunk("Hello", "السلام عليكم", "fa") + # No Persian-specific chars, no other Arabic-script variant chars + # → should pass the variant check (because no discrimination signal). + # This is a known limitation: pure Arabic indistinguishable from pure Persian. + # We accept that the variant check only flags MISMATCHES with discriminating chars. + assert "wrong_arabic_variant" not in r.issues + + def test_urdu_for_persian_target_fails(self): + # Urdu with ٹ ڈ should fail when target is Persian. + r = evaluate_chunk("Hello", "السلام ٹڈ", "fa") + assert r.passed is False + assert "wrong_arabic_variant" in r.issues + + def test_pashto_for_arabic_target_fails(self): + # Pashto with ټډړ should fail when target is Arabic. + r = evaluate_chunk("Hello", "السلام ټډړ", "ar") + assert r.passed is False + assert "wrong_arabic_variant" in r.issues + + +# ---------- Edge cases ---------- + +class TestEdgeCases: + def test_empty_translation_passes(self): + # Empty translations are skipped (the provider may have legitimately + # produced nothing — e.g. an empty cell). + r = evaluate_chunk("Hello", "", "fr") + assert r.passed is True + assert "empty_translation" in r.issues + + def test_whitespace_only_passes(self): + r = evaluate_chunk("Hello", " \n ", "fr") + assert r.passed is True + assert "empty_translation" in r.issues + + def test_none_translation_passes(self): + r = evaluate_chunk("Hello", None, "fr") + assert r.passed is True + + def test_numbers_only_passes(self): + r = evaluate_chunk("Price: 100", "100", "fr") + assert r.passed is True + + def test_unknown_target_lang_falls_back(self): + # Unknown lang → latin → no script check, just length/leak checks. + r = evaluate_chunk("Hello", "Bonjour", "xx") + assert r.passed is True + + def test_mixed_brands_latin_target(self): + # "iPhone 15 Pro Max" is mostly Latin + digits — fine for fr. + r = evaluate_chunk("Buy iPhone 15", "Acheter iPhone 15", "fr") + assert r.passed is True + + +# ---------- Length sanity ---------- + +class TestLengthIssues: + def test_huge_translation_flagged_via_repetition(self): + # Source too short for length ratio, but the "x"*1000 repetition + # is caught by the pattern/repetition check, not length. + r = evaluate_chunk("Short", "x" * 1000, "fr") + assert "repetition_hallucination" in r.issues + + def test_huge_translation_with_long_source_flagged(self): + # Long enough source → length ratio kicks in → length_outlier. + src = "A" * 200 + r = evaluate_chunk(src, "x" * 1000, "fr") + assert "length_outlier" in r.issues + + def test_tiny_translation_flagged(self): + r = evaluate_chunk("A" * 100, "ok", "fr") + # ratio is 2/100 = 0.02, well below 0.15 + assert "truncation_suspect" in r.issues + + def test_normal_translation_no_length_issue(self): + r = evaluate_chunk("Hello world, how are you?", "Bonjour le monde, comment allez-vous?", "fr") + assert "length_outlier" not in r.issues + assert "truncation_suspect" not in r.issues + + def test_short_source_skips_ratio(self): + r = evaluate_chunk("Hi", "ok", "fr") + # source too short for ratio check + assert "length_outlier" not in r.issues + assert "truncation_suspect" not in r.issues + + def test_numeric_source_skips_length_check(self): + # Source is mostly digits — translation can also be short. + r = evaluate_chunk("Price: 100", "100", "fr") + assert "truncation_suspect" not in r.issues + assert "length_outlier" not in r.issues + + +# ---------- Pattern leak / repetition ---------- + +class TestPatternLeak: + def test_prompt_leak_english_detected(self): + r = evaluate_chunk("Hello", "Translation: Bonjour le monde", "fr") + assert "prompt_leak" in r.issues + + def test_prompt_leak_french_detected(self): + r = evaluate_chunk("Hello", "Voici la traduction : Bonjour", "fr") + assert "prompt_leak" in r.issues + + def test_prompt_leak_chinese_detected(self): + r = evaluate_chunk("Hello", "翻译：你好", "zh") + assert "prompt_leak" in r.issues + + def test_repetition_hallucination_detected(self): + r = evaluate_chunk("Hello", "the the the the the the", "fr") + assert "repetition_hallucination" in r.issues + + def test_normal_text_no_leak(self): + r = evaluate_chunk("Hello", "Bonjour le monde", "fr") + assert "prompt_leak" not in r.issues + assert "repetition_hallucination" not in r.issues + + +# ---------- Document-level aggregation ---------- + +class TestEvaluateDocument: + def test_all_good(self): + result = evaluate_document( + ["Hello", "World", "Good morning"], + ["Bonjour", "Monde", "Bonjour"], + "fr", + ) + assert result.passed is True + assert result.failed_chunk_count == 0 + assert result.chunk_count == 3 + + def test_some_failures(self): + result = evaluate_document( + ["Hello", "World", "Good morning"], + ["Bonjour", "مرحبا", "Bonjour"], + "fr", + ) + assert result.passed is False + assert result.failed_chunk_count == 1 + assert "wrong_script" in result.issues + assert len(result.samples) == 1 + + def test_empty_lists(self): + result = evaluate_document([], [], "fr") + assert result.passed is True + assert result.chunk_count == 0 + assert result.score == 0.0 + + def test_sample_size_caps_samples(self): + # 10 failures, sample_size=3 + result = evaluate_document( + ["hi"] * 10, + ["مرحبا"] * 10, + "fr", + sample_size=3, + ) + assert result.failed_chunk_count == 10 + assert len(result.samples) == 3 + + def test_score_proportional(self): + # 4 chunks, 1 fails → score should be ~0.75 (3/4 passed checks) + result = evaluate_document( + ["hi", "hi", "hi", "hi"], + ["bonjour", "bonjour", "مرحبا", "bonjour"], + "fr", + ) + assert result.failed_chunk_count == 1 + assert result.score < 1.0 + assert result.score > 0.0 + + +# ---------- detect_arabic_variant direct ---------- + +class TestDetectArabicVariantDirect: + def test_empty_text(self): + r = detect_arabic_variant("", "fa") + assert r["verdict"] == "skip" + + def test_pure_persian_for_persian(self): + r = detect_arabic_variant("سلام چطوری", "fa") + assert r["verdict"] == "pass" + + def test_pure_persian_for_arabic(self): + r = detect_arabic_variant("سلام چطوری", "ar") + assert r["verdict"] == "fail" + assert "fa" in r["detected_variants"] + + def test_pure_urdu_for_persian(self): + r = detect_arabic_variant("السلام ٹڈ", "fa") + assert r["verdict"] == "fail" + assert "ur" in r["detected_variants"] + + def test_non_arabic_text(self): + r = detect_arabic_variant("Bonjour le monde", "fa") + # Not in Arabic script → skip + assert r["verdict"] == "skip" diff --git a/wordly.art---traduction-de-documents/package.json b/wordly.art---traduction-de-documents/package.json index ad972ef..2ff4879 100644 --- a/wordly.art---traduction-de-documents/package.json +++ b/wordly.art---traduction-de-documents/package.json @@ -8,7 +8,8 @@ "build": "vite build", "preview": "vite preview", "clean": "rm -rf dist server.js", - "lint": "tsc --noEmit" + "lint": "tsc --noEmit", + "test:quality": "node --experimental-strip-types --test tests/utils/scriptDetector.test.ts" }, "dependencies": { "@google/genai": "^1.29.0", diff --git a/wordly.art---traduction-de-documents/src/utils/scriptDetector.ts b/wordly.art---traduction-de-documents/src/utils/scriptDetector.ts new file mode 100644 index 0000000..81e51f6 --- /dev/null +++ b/wordly.art---traduction-de-documents/src/utils/scriptDetector.ts @@ -0,0 +1,739 @@ +/** + * L0 quality detector — TypeScript mirror of services/quality/ in Python. + * + * Detects: + * - wrong_script: translation is in the wrong script for the target language + * (e.g. user asks French, model returns Arabic — language confusion) + * - wrong_arabic_variant: for Arabic-script targets (ar/fa/ur/...), + * the translation uses characters of a DIFFERENT Arabic-script language + * (e.g. user asks Persian, model returns Urdu) + * - length_outlier / truncation_suspect: translation is wildly different + * in length from the source + * - prompt_leak: translation starts with "Translation:" or similar + * - repetition_hallucination: "xxx..." or "the the the the" pattern + * + * Zero dependencies. Works in browser AND in Node.js (for tests). + * Designed to be ADDITIVE — never blocks the translation flow. + */ + +// ---------- Types ---------- + +export interface QualityCheckResult { + passed: boolean; + score: number; // 0.0 to 1.0 + issues: string[]; + detectedScript: string | null; + expectedScript: string | null; + details: Record; +} + +export interface DocumentQualityResult { + passed: boolean; + score: number; + chunkCount: number; + failedChunkCount: number; + issues: Record; + samples: Array<{ + index: number; + issues: string[]; + sourcePreview: string; + translatedPreview: string; + details: Record; + }>; +} + +// ---------- Unicode ranges per script ---------- +// Mirrors services/quality/config.py in Python. + +type Range = readonly [number, number]; // [start, end] inclusive + +const UNICODE_RANGES: Record = { + cyrillic: [ + [0x0400, 0x04ff], + [0x0500, 0x052f], + ], + greek: [ + [0x0370, 0x03ff], + ], + arabic: [ + [0x0600, 0x06ff], + [0x0750, 0x077f], + [0x08a0, 0x08ff], + ], + hebrew: [ + [0x0590, 0x05ff], + ], + devanagari: [ + [0x0900, 0x097f], + ], + bengali: [ + [0x0980, 0x09ff], + ], + tamil: [ + [0x0b80, 0x0bff], + ], + telugu: [ + [0x0c00, 0x0c7f], + ], + kannada: [ + [0x0c80, 0x0cff], + ], + malayalam: [ + [0x0d00, 0x0d7f], + ], + sinhala: [ + [0x0d80, 0x0dff], + ], + gujarati: [ + [0x0a80, 0x0aff], + ], + gurmukhi: [ + [0x0a00, 0x0a7f], + ], + thai: [ + [0x0e00, 0x0e7f], + ], + lao: [ + [0x0e80, 0x0eff], + ], + burmese: [ + [0x1000, 0x109f], + ], + khmer: [ + [0x1780, 0x17ff], + ], + cjk: [ + [0x4e00, 0x9fff], + [0x3400, 0x4dbf], + ], + hiragana_katakana: [ + [0x3040, 0x309f], + [0x30a0, 0x30ff], + ], + hangul: [ + [0xac00, 0xd7af], + [0x1100, 0x11ff], + [0xa960, 0xa97f], + ], + georgian: [ + [0x10a0, 0x10ff], + ], + armenian: [ + [0x0530, 0x058f], + ], + ethiopic: [ + [0x1200, 0x137f], + [0x1380, 0x139f], + ], + tibetan: [ + [0x0f00, 0x0fff], + ], + thaana: [ + [0x0780, 0x07bf], + ], + latin: [], +}; + +// ---------- Language → script mapping ---------- + +const LANG_TO_SCRIPT: Record = { + // Latin-script languages + en: 'latin', fr: 'latin', de: 'latin', es: 'latin', it: 'latin', + pt: 'latin', nl: 'latin', pl: 'latin', tr: 'latin', vi: 'latin', + id: 'latin', ms: 'latin', ro: 'latin', cs: 'latin', sv: 'latin', + da: 'latin', fi: 'latin', no: 'latin', nb: 'latin', nn: 'latin', + hu: 'latin', sk: 'latin', sl: 'latin', lt: 'latin', lv: 'latin', + et: 'latin', sq: 'latin', az: 'latin', uz: 'latin', kk: 'latin', + ky: 'latin', tk: 'latin', sw: 'latin', eu: 'latin', gl: 'latin', + is: 'latin', ga: 'latin', mt: 'latin', ca: 'latin', hr: 'latin', + bs: 'latin', af: 'latin', cy: 'latin', lb: 'latin', fo: 'latin', + br: 'latin', co: 'latin', fy: 'latin', gd: 'latin', gu: 'latin', + ht: 'latin', haw: 'latin', hmn: 'latin', jv: 'latin', ku: 'latin', + mg: 'latin', mi: 'latin', mn: 'latin', nso: 'latin', ny: 'latin', + oc: 'latin', os: 'latin', ps: 'latin', qu: 'latin', rw: 'latin', + sc: 'latin', si: 'latin', sm: 'latin', sn: 'latin', so: 'latin', + st: 'latin', su: 'latin', tg: 'latin', tt: 'latin', ty: 'latin', + ug: 'latin', vo: 'latin', wa: 'latin', wo: 'latin', xh: 'latin', + yi: 'latin', zu: 'latin', + // Cyrillic overrides + ru: 'cyrillic', uk: 'cyrillic', be: 'cyrillic', sr: 'cyrillic', + mk: 'cyrillic', bg: 'cyrillic', ce: 'cyrillic', cv: 'cyrillic', + sah: 'cyrillic', udm: 'cyrillic', rue: 'cyrillic', ab: 'cyrillic', + // Greek + el: 'greek', + // Arabic-script languages + ar: 'arabic', fa: 'arabic', ur: 'arabic', ps: 'arabic', + ku: 'arabic', sd: 'arabic', ckb: 'arabic', bal: 'arabic', + bqi: 'arabic', glk: 'arabic', mzn: 'arabic', + // Hebrew & Yiddish + he: 'hebrew', + yi: 'hebrew', // Yiddish uses Hebrew script, not Arabic + // Thaana (Maldivian) + dv: 'thaana', // Maldivian uses Thaana, not Arabic + // Indian scripts + hi: 'devanagari', ne: 'devanagari', mr: 'devanagari', sa: 'devanagari', + mai: 'devanagari', bho: 'devanagari', awa: 'devanagari', + bn: 'bengali', as: 'bengali', + ta: 'tamil', + te: 'telugu', + kn: 'kannada', + ml: 'malayalam', + si: 'sinhala', + gu: 'gujarati', + pa: 'gurmukhi', + // SE Asia + th: 'thai', + lo: 'lao', + my: 'burmese', + km: 'khmer', + // East Asia + zh: 'cjk', 'zh-cn': 'cjk', 'zh-tw': 'cjk', 'zh-hk': 'cjk', + ja: 'hiragana_katakana', + ko: 'hangul', + // Others + ka: 'georgian', + hy: 'armenian', + am: 'ethiopic', ti: 'ethiopic', gez: 'ethiopic', tig: 'ethiopic', + bo: 'tibetan', dz: 'tibetan', +}; + +// ---------- Discriminating characters for Arabic-script languages ---------- + +const DISCRIMINATING_CHARS: Record> = { + fa: new Set('پچژگ'), // Persian + ur: new Set('ٹڈڑے'), // Urdu + ps: new Set('ټډړږښ'), // Pashto + ku: new Set('ڕێ'), // Kurdish + ckb: new Set('ڕێ'), // Sorani Kurdish + sd: new Set('ٿ'), // Sindhi + ug: new Set('ۇۆې'), // Uyghur + bal: new Set('ێ'), // Balochi +}; + +// ---------- Thresholds ---------- + +const MIN_RATIO_IN_SCRIPT = 0.60; + +// ---------- Helpers ---------- + +/** Get the script id for a language code. */ +export function getScript(langCode: string | null | undefined): string { + if (!langCode) return 'latin'; + return LANG_TO_SCRIPT[langCode.toLowerCase()] ?? 'latin'; +} + +/** True if the language uses an Arabic-script block. */ +export function isArabicScriptLang(langCode: string | null | undefined): boolean { + return getScript(langCode) === 'arabic'; +} + +/** Get the Unicode ranges for a script id. Empty array for 'latin'. */ +function getRanges(scriptId: string): readonly Range[] { + return UNICODE_RANGES[scriptId] ?? []; +} + +/** Iterate code points of a string (handles surrogate pairs correctly). */ +function* codePoints(text: string): Generator { + for (const ch of text) { + yield ch.codePointAt(0) ?? 0; + } +} + +/** Check if a code point falls in any of the (start, end) ranges. */ +function isInRanges(code: number, ranges: readonly Range[]): boolean { + for (const [start, end] of ranges) { + if (code >= start && code <= end) return true; + } + return false; +} + +const LETTER_REGEX = /\p{L}/u; + +/** Count alphabetic characters (using Unicode property escapes). */ +function countLetters(text: string): number { + let count = 0; + for (const ch of text) { + if (LETTER_REGEX.test(ch)) count++; + } + return count; +} + +/** Count how many alphabetic characters fall within the given ranges. */ +function countInScript(text: string, ranges: readonly Range[]): number { + if (ranges.length === 0) { + // Latin / unknown — all letters match. + return countLetters(text); + } + let count = 0; + for (const ch of text) { + if (!LETTER_REGEX.test(ch)) continue; + const cp = ch.codePointAt(0) ?? 0; + if (isInRanges(cp, ranges)) count++; + } + return count; +} + +// ---------- Arabic-script variant detection ---------- + +export interface ArabicVariantResult { + verdict: 'pass' | 'fail' | 'skip'; + claimedLang: string | null; + detectedVariants: string[]; + arabicRatio?: number; + reason: string; +} + +export function detectArabicVariant( + text: string, + claimedLang: string | null, +): ArabicVariantResult { + if (!text || !text.trim()) { + return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'empty text' }; + } + + const arabicRanges = getRanges('arabic'); + const letters = countLetters(text); + if (letters === 0) { + return { verdict: 'skip', claimedLang, detectedVariants: [], reason: 'no letters' }; + } + + const inArabic = countInScript(text, arabicRanges); + const arabicRatio = inArabic / letters; + + if (arabicRatio < MIN_RATIO_IN_SCRIPT) { + return { + verdict: 'skip', + claimedLang, + detectedVariants: [], + arabicRatio: round(arabicRatio, 3), + reason: 'not in Arabic script', + }; + } + + if (!isArabicScriptLang(claimedLang)) { + return { + verdict: 'skip', + claimedLang, + detectedVariants: [], + arabicRatio: round(arabicRatio, 3), + reason: 'target is not an Arabic-script language', + }; + } + + // Text is Arabic-script AND target is Arabic-script: check the variant. + const detected = new Set(); + for (const [langCode, chars] of Object.entries(DISCRIMINATING_CHARS)) { + if (chars.size === 0) continue; + for (const ch of text) { + if (chars.has(ch)) { + detected.add(langCode); + break; + } + } + } + + if (detected.size > 0 && claimedLang && !detected.has(claimedLang.toLowerCase())) { + return { + verdict: 'fail', + claimedLang, + detectedVariants: Array.from(detected).sort(), + arabicRatio: round(arabicRatio, 3), + reason: `target=${claimedLang} but text contains characters typical of ${Array.from(detected).sort().join(', ')}`, + }; + } + + return { + verdict: 'pass', + claimedLang, + detectedVariants: detected.size > 0 ? Array.from(detected).sort() : (claimedLang ? [claimedLang] : []), + arabicRatio: round(arabicRatio, 3), + reason: 'ok', + }; +} + +// ---------- Length check ---------- + +const RATIO_MAX = 3.5; +const RATIO_MIN = 0.15; +const ABSOLUTE_MIN_LENGTH = 2; +const MIN_SOURCE_LENGTH_FOR_RATIO = 20; + +export interface LengthCheckResult { + issue: string | null; + ratio: number | null; + sourceLength: number; + translatedLength: number; + note?: string; +} + +export function lengthCheck(sourceText: string, translatedText: string): LengthCheckResult { + if (!sourceText) { + return { + issue: null, + ratio: null, + sourceLength: 0, + translatedLength: (translatedText || '').length, + }; + } + + const srcLen = sourceText.trim().length; + const transLen = (translatedText || '').trim().length; + + if (transLen === 0) { + return { + issue: 'truncation_suspect', + ratio: 0, + sourceLength: srcLen, + translatedLength: transLen, + }; + } + + if (isMostlyNumeric(sourceText)) { + return { + issue: null, + ratio: null, + sourceLength: srcLen, + translatedLength: transLen, + note: 'skipped: numeric source', + }; + } + + if (srcLen < MIN_SOURCE_LENGTH_FOR_RATIO) { + return { + issue: null, + ratio: null, + sourceLength: srcLen, + translatedLength: transLen, + }; + } + + const ratio = transLen / srcLen; + + if (ratio > RATIO_MAX) { + return { + issue: 'length_outlier', + ratio: round(ratio, 2), + sourceLength: srcLen, + translatedLength: transLen, + }; + } + if (ratio < RATIO_MIN) { + return { + issue: 'truncation_suspect', + ratio: round(ratio, 2), + sourceLength: srcLen, + translatedLength: transLen, + }; + } + + return { + issue: null, + ratio: round(ratio, 2), + sourceLength: srcLen, + translatedLength: transLen, + }; +} + +function isMostlyNumeric(text: string): boolean { + if (!text) return false; + const chars: string[] = []; + for (const ch of text) { + if (!/\s/.test(ch)) chars.push(ch); + } + if (chars.length === 0) return false; + const digitCount = chars.filter((c) => /\d/.test(c)).length; + return digitCount / chars.length >= 0.5; +} + +// ---------- Pattern leak / repetition ---------- + +const LEAK_PREFIX_PATTERNS: RegExp[] = [ + /^(translation|translated text|here is the translation|here'?s the translation)\s*[:：-]/i, + /^(voici (la |ma )?traduction|traduction\s*[:：-])\b/i, + /^(原文|译|翻译|译为|以下是)\s*[:：]?/u, + /^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)/i, + /^(\*\*|__|#)\s*translation/i, + /^translated from\s+\w+\s+to\s+\w+\s*[:：-]/i, +]; + +const REPETITION_THRESHOLD = 5; +const CHAR_REPETITION_THRESHOLD = 20; + +export interface PatternCheckResult { + issue: string | null; + matchedPattern: string | null; + repetitionCount: number | null; +} + +export function patternCheck(text: string): PatternCheckResult { + if (!text || !text.trim()) { + return { issue: null, matchedPattern: null, repetitionCount: null }; + } + + const stripped = text.trimStart(); + + // 1. Prompt leak + for (const pat of LEAK_PREFIX_PATTERNS) { + if (pat.test(stripped)) { + return { + issue: 'prompt_leak', + matchedPattern: pat.source, + repetitionCount: null, + }; + } + } + + // 2. Token-level repetition + const tokens = stripped.split(/\s+/).filter((t) => t.length > 0); + const tokenRep = maxConsecutiveTokenRepetition(tokens); + if (tokenRep >= REPETITION_THRESHOLD) { + return { + issue: 'repetition_hallucination', + matchedPattern: null, + repetitionCount: tokenRep, + }; + } + + // 3. Character-level repetition + const charRep = maxConsecutiveCharRepetition(stripped); + if (charRep >= CHAR_REPETITION_THRESHOLD) { + return { + issue: 'repetition_hallucination', + matchedPattern: null, + repetitionCount: charRep, + }; + } + + return { + issue: null, + matchedPattern: null, + repetitionCount: Math.max(tokenRep, charRep) || null, + }; +} + +function maxConsecutiveTokenRepetition(tokens: string[]): number { + if (tokens.length === 0) return 0; + const norm = tokens.map((t) => t.toLowerCase().replace(/[.,!?;:"'`()\[\]{}]/g, '')); + let maxRun = 1; + let currentRun = 1; + for (let i = 1; i < norm.length; i++) { + if (norm[i] && norm[i] === norm[i - 1]) { + currentRun++; + if (currentRun > maxRun) maxRun = currentRun; + } else { + currentRun = 1; + } + } + return maxRun; +} + +function maxConsecutiveCharRepetition(text: string): number { + if (!text) return 0; + let maxRun = 1; + let currentRun = 1; + for (let i = 1; i < text.length; i++) { + if (text[i] === text[i - 1] && !/\s/.test(text[i])) { + currentRun++; + if (currentRun > maxRun) maxRun = currentRun; + } else { + currentRun = 1; + } + } + return maxRun; +} + +// ---------- Heuristic actual-script detection (for diagnostics) ---------- + +const SCRIPT_DETECTION_ORDER: readonly string[] = [ + 'hiragana_katakana', 'hangul', 'cjk', 'thai', 'lao', 'burmese', + 'khmer', 'devanagari', 'bengali', 'tamil', 'telugu', 'kannada', + 'malayalam', 'sinhala', 'gujarati', 'gurmukhi', + 'arabic', 'hebrew', 'cyrillic', 'greek', 'armenian', 'georgian', + 'ethiopic', 'tibetan', 'thaana', +]; + +function detectActualScript(text: string): string { + const letters = countLetters(text); + if (letters === 0) return 'unknown'; + for (const scriptId of SCRIPT_DETECTION_ORDER) { + const ranges = getRanges(scriptId); + const inScript = countInScript(text, ranges); + if (inScript / letters > 0.4) return scriptId; + } + return 'latin'; +} + +// ---------- Main per-chunk evaluation ---------- + +export function evaluateChunk( + sourceText: string, + translatedText: string | null | undefined, + targetLang: string | null | undefined, +): QualityCheckResult { + if (translatedText === null || translatedText === undefined) { + return { + passed: true, + score: 0, + issues: ['empty_translation'], + detectedScript: null, + expectedScript: null, + details: { reason: 'translation is null/undefined' }, + }; + } + + const text = translatedText.trim(); + if (!text) { + return { + passed: true, + score: 0, + issues: ['empty_translation'], + detectedScript: null, + expectedScript: null, + details: { reason: 'translation is empty or whitespace-only' }, + }; + } + + const targetLangLower = (targetLang || '').toLowerCase() || null; + const issues: string[] = []; + const details: Record = {}; + + // --- Script detection --- + const expectedScript = getScript(targetLangLower); + const expectedRanges = getRanges(expectedScript); + const letters = countLetters(text); + + let scriptScore: number; + let detectedScript: string | null; + + if (letters === 0) { + scriptScore = 1.0; + detectedScript = expectedScript; + details.script_check = 'skipped: no alphabetic characters'; + } else { + detectedScript = detectActualScript(text); + const inExpected = countInScript(text, expectedRanges); + scriptScore = inExpected / letters; + + details.script_score = round(scriptScore, 3); + details.letters_in_text = letters; + details.letters_in_script = inExpected; + details.detected_script = detectedScript; + details.expected_script = expectedScript; + details.min_ratio = MIN_RATIO_IN_SCRIPT; + + if (expectedScript !== 'latin' && expectedRanges.length > 0) { + // Specific non-Latin target. + if (scriptScore < MIN_RATIO_IN_SCRIPT) { + issues.push('wrong_script'); + details.reason = `only ${Math.round(scriptScore * 100)}% of letters match ${expectedScript} script; text appears to be in ${detectedScript}`; + } + } else { + // Latin target. If detected script is clearly non-Latin, fail. + if (detectedScript && detectedScript !== 'latin' && detectedScript !== 'unknown') { + const nonLatinRanges = getRanges(detectedScript); + const inDetected = countInScript(text, nonLatinRanges); + const nonLatinConfidence = inDetected / letters; + if (nonLatinConfidence >= 0.7) { + issues.push('wrong_script'); + details.reason = `target is Latin but ${Math.round(nonLatinConfidence * 100)}% of letters are in ${detectedScript} script — language confusion`; + } + } + } + } + + // --- Arabic-script variant detection --- + if (isArabicScriptLang(targetLangLower)) { + const variantResult = detectArabicVariant(text, targetLangLower); + details.arabic_variant = variantResult; + if (variantResult.verdict === 'fail') { + issues.push('wrong_arabic_variant'); + } + } + + // --- Length sanity --- + const lengthResult = lengthCheck(sourceText, text); + details.length = lengthResult; + if (lengthResult.issue) { + issues.push(lengthResult.issue); + } + + // --- Pattern leak / repetition --- + const leakResult = patternCheck(text); + details.pattern_check = leakResult; + if (leakResult.issue) { + issues.push(leakResult.issue); + } + + // --- Aggregate --- + const passed = issues.length === 0; + const nChecks = 3; + const nFailed = issues.filter((issue) => + ['wrong_script', 'wrong_arabic_variant', 'length_outlier', 'truncation_suspect', 'prompt_leak', 'repetition_hallucination'].includes(issue), + ).length; + const score = Math.max(0, 1 - nFailed / nChecks); + + return { + passed, + score: round(score, 3), + issues, + detectedScript, + expectedScript, + details, + }; +} + +// ---------- Document-level aggregation ---------- + +export function evaluateDocument( + sourceChunks: string[], + translatedChunks: string[], + targetLang: string | null | undefined, + sampleSize: number = 50, +): DocumentQualityResult { + const n = Math.min(sourceChunks.length, translatedChunks.length); + const chunkResults: QualityCheckResult[] = []; + const issuesCount: Record = {}; + const samples: DocumentQualityResult['samples'] = []; + let scoreSum = 0; + let failedCount = 0; + + for (let i = 0; i < n; i++) { + const r = evaluateChunk(sourceChunks[i], translatedChunks[i], targetLang); + chunkResults.push(r); + scoreSum += r.score; + for (const issue of r.issues) { + issuesCount[issue] = (issuesCount[issue] || 0) + 1; + } + if (!r.passed) { + failedCount++; + if (samples.length < sampleSize) { + samples.push({ + index: i, + issues: r.issues, + sourcePreview: (sourceChunks[i] || '').slice(0, 80), + translatedPreview: (translatedChunks[i] || '').slice(0, 80), + details: r.details, + }); + } + } + } + + const meanScore = n > 0 ? scoreSum / n : 0; + const passed = failedCount === 0; + + return { + passed, + score: round(meanScore, 3), + chunkCount: n, + failedChunkCount: failedCount, + issues: issuesCount, + samples, + }; +} + +// ---------- Utilities ---------- + +function round(value: number, decimals: number): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} diff --git a/wordly.art---traduction-de-documents/tests/utils/scriptDetector.test.ts b/wordly.art---traduction-de-documents/tests/utils/scriptDetector.test.ts new file mode 100644 index 0000000..607fd32 --- /dev/null +++ b/wordly.art---traduction-de-documents/tests/utils/scriptDetector.test.ts @@ -0,0 +1,421 @@ +/** + * Tests for src/utils/scriptDetector.ts + * + * Run with: + * npx tsx --test tests/utils/scriptDetector.test.ts + * + * Or in package.json scripts: + * "test:quality": "tsx --test tests/utils/scriptDetector.test.ts" + */ +import { describe, it } from 'node:test'; +import { strict as assert } from 'node:assert'; + +import { + getScript, + isArabicScriptLang, + evaluateChunk, + evaluateDocument, + detectArabicVariant, + lengthCheck, + patternCheck, +} from '../../src/utils/scriptDetector.ts'; + +describe('getScript', () => { + it('maps cyrillic languages', () => { + assert.equal(getScript('ru'), 'cyrillic'); + assert.equal(getScript('uk'), 'cyrillic'); + assert.equal(getScript('be'), 'cyrillic'); + }); + + it('maps latin languages', () => { + assert.equal(getScript('en'), 'latin'); + assert.equal(getScript('fr'), 'latin'); + assert.equal(getScript('de'), 'latin'); + assert.equal(getScript('vi'), 'latin'); + }); + + it('maps CJK languages', () => { + assert.equal(getScript('zh'), 'cjk'); + assert.equal(getScript('zh-cn'), 'cjk'); + }); + + it('maps Japanese to hiragana_katakana', () => { + assert.equal(getScript('ja'), 'hiragana_katakana'); + }); + + it('maps Korean to hangul', () => { + assert.equal(getScript('ko'), 'hangul'); + }); + + it('maps Yiddish to hebrew (not arabic)', () => { + assert.equal(getScript('yi'), 'hebrew'); + }); + + it('maps Maldivian to thaana (not arabic)', () => { + assert.equal(getScript('dv'), 'thaana'); + }); + + it('falls back to latin for unknown', () => { + assert.equal(getScript('xx'), 'latin'); + assert.equal(getScript(''), 'latin'); + assert.equal(getScript(null), 'latin'); + }); + + it('is case-insensitive', () => { + assert.equal(getScript('FR'), 'latin'); + assert.equal(getScript('ZH-CN'), 'cjk'); + }); +}); + +describe('isArabicScriptLang', () => { + it('returns true for Arabic-script langs', () => { + assert.equal(isArabicScriptLang('ar'), true); + assert.equal(isArabicScriptLang('fa'), true); + assert.equal(isArabicScriptLang('ur'), true); + assert.equal(isArabicScriptLang('ckb'), true); + }); + + it('returns false for non-Arabic langs', () => { + assert.equal(isArabicScriptLang('fr'), false); + assert.equal(isArabicScriptLang('he'), false); + assert.equal(isArabicScriptLang('en'), false); + }); +}); + +// ---------- Happy path ---------- + +describe('evaluateChunk — correct script', () => { + it('russian', () => { + const r = evaluateChunk('Hello world', 'Привет мир', 'ru'); + assert.equal(r.passed, true); + assert.ok(!r.issues.includes('wrong_script')); + }); + + it('chinese', () => { + const r = evaluateChunk('Hello', '你好世界', 'zh'); + assert.equal(r.passed, true); + }); + + it('arabic', () => { + const r = evaluateChunk('Hello', 'مرحبا بالعالم', 'ar'); + assert.equal(r.passed, true); + }); + + it('persian (with unique chars)', () => { + const r = evaluateChunk('Hello', 'سلام چطوری؟ من پژوهشگر هستم', 'fa'); + assert.equal(r.passed, true); + }); + + it('french', () => { + const r = evaluateChunk('Hello', 'Bonjour le monde', 'fr'); + assert.equal(r.passed, true); + }); + + it('hebrew', () => { + const r = evaluateChunk('Hello', 'שלום עולם', 'he'); + assert.equal(r.passed, true); + }); + + it('korean', () => { + const r = evaluateChunk('Hello', '안녕하세요 세계', 'ko'); + assert.equal(r.passed, true); + }); + + it('japanese', () => { + const r = evaluateChunk('Hello', 'こんにちは世界', 'ja'); + assert.equal(r.passed, true); + }); + + it('hindi', () => { + const r = evaluateChunk('Hello', 'नमस्ते दुनिया', 'hi'); + assert.equal(r.passed, true); + }); + + it('thai', () => { + const r = evaluateChunk('Hello', 'สวัสดีชาวโลก', 'th'); + assert.equal(r.passed, true); + }); + + it('greek', () => { + const r = evaluateChunk('Hello', 'Γεια σας κόσμε', 'el'); + assert.equal(r.passed, true); + }); +}); + +// ---------- Wrong script detection ---------- + +describe('evaluateChunk — wrong script', () => { + it('french text for japanese target', () => { + const r = evaluateChunk('Hello', 'Bonjour le monde', 'ja'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_script')); + }); + + it('arabic text for french target (language confusion)', () => { + const r = evaluateChunk('Hello', 'مرحبا بالعالم', 'fr'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_script')); + }); + + it('russian text for english target', () => { + const r = evaluateChunk('Hello', 'Привет мир', 'en'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_script')); + }); + + it('chinese text for korean target', () => { + const r = evaluateChunk('Hello', '你好世界', 'ko'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_script')); + }); +}); + +// ---------- Arabic variant discrimination ---------- + +describe('evaluateChunk — Arabic variant discrimination', () => { + it('persian text for arabic target fails', () => { + const r = evaluateChunk('Hello', 'سلام چطوری؟', 'ar'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_arabic_variant')); + }); + + it('urdu text for persian target fails', () => { + const r = evaluateChunk('Hello', 'السلام ٹڈ', 'fa'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_arabic_variant')); + }); + + it('pashto text for arabic target fails', () => { + const r = evaluateChunk('Hello', 'السلام ټډړ', 'ar'); + assert.equal(r.passed, false); + assert.ok(r.issues.includes('wrong_arabic_variant')); + }); +}); + +// ---------- Edge cases ---------- + +describe('evaluateChunk — edge cases', () => { + it('empty translation passes (skipped)', () => { + const r = evaluateChunk('Hello', '', 'fr'); + assert.equal(r.passed, true); + assert.ok(r.issues.includes('empty_translation')); + }); + + it('whitespace-only translation passes', () => { + const r = evaluateChunk('Hello', ' \n ', 'fr'); + assert.equal(r.passed, true); + }); + + it('null translation passes', () => { + const r = evaluateChunk('Hello', null, 'fr'); + assert.equal(r.passed, true); + }); + + it('undefined translation passes', () => { + const r = evaluateChunk('Hello', undefined, 'fr'); + assert.equal(r.passed, true); + }); + + it('numbers only pass for fr target', () => { + const r = evaluateChunk('Price: 100', '100', 'fr'); + assert.equal(r.passed, true); + }); + + it('unknown target lang falls back to latin', () => { + const r = evaluateChunk('Hello', 'Bonjour', 'xx'); + assert.equal(r.passed, true); + }); +}); + +// ---------- Length ---------- + +describe('evaluateChunk — length', () => { + it('huge translation flagged via repetition (source too short)', () => { + const r = evaluateChunk('Short', 'x'.repeat(1000), 'fr'); + assert.ok(r.issues.includes('repetition_hallucination')); + }); + + it('huge translation with long source flagged as length', () => { + const src = 'A'.repeat(200); + const r = evaluateChunk(src, 'x'.repeat(1000), 'fr'); + assert.ok(r.issues.includes('length_outlier')); + }); + + it('tiny translation flagged', () => { + const r = evaluateChunk('A'.repeat(100), 'ok', 'fr'); + assert.ok(r.issues.includes('truncation_suspect')); + }); + + it('numeric source skips length check', () => { + const r = evaluateChunk('Price: 100', '100', 'fr'); + assert.ok(!r.issues.includes('truncation_suspect')); + assert.ok(!r.issues.includes('length_outlier')); + }); +}); + +// ---------- Pattern leak / repetition ---------- + +describe('evaluateChunk — pattern leak', () => { + it('english prompt leak', () => { + const r = evaluateChunk('Hello', 'Translation: Bonjour le monde', 'fr'); + assert.ok(r.issues.includes('prompt_leak')); + }); + + it('french prompt leak', () => { + const r = evaluateChunk('Hello', 'Voici la traduction : Bonjour', 'fr'); + assert.ok(r.issues.includes('prompt_leak')); + }); + + it('chinese prompt leak', () => { + const r = evaluateChunk('Hello', '翻译：你好', 'zh'); + assert.ok(r.issues.includes('prompt_leak')); + }); + + it('token repetition detected', () => { + const r = evaluateChunk('Hello', 'the the the the the the the', 'fr'); + assert.ok(r.issues.includes('repetition_hallucination')); + }); + + it('char repetition detected', () => { + const r = evaluateChunk('Hello', 'x'.repeat(30), 'fr'); + assert.ok(r.issues.includes('repetition_hallucination')); + }); + + it('normal text has no leak', () => { + const r = evaluateChunk('Hello', 'Bonjour le monde', 'fr'); + assert.ok(!r.issues.includes('prompt_leak')); + assert.ok(!r.issues.includes('repetition_hallucination')); + }); +}); + +// ---------- Document-level ---------- + +describe('evaluateDocument', () => { + it('all good', () => { + const result = evaluateDocument( + ['Hello', 'World', 'Good morning'], + ['Bonjour', 'Monde', 'Bonjour'], + 'fr', + ); + assert.equal(result.passed, true); + assert.equal(result.chunkCount, 3); + assert.equal(result.failedChunkCount, 0); + }); + + it('some failures', () => { + const result = evaluateDocument( + ['Hello', 'World', 'Good morning'], + ['Bonjour', 'مرحبا', 'Bonjour'], + 'fr', + ); + assert.equal(result.passed, false); + assert.equal(result.failedChunkCount, 1); + assert.ok('wrong_script' in result.issues); + assert.equal(result.samples.length, 1); + }); + + it('empty lists', () => { + const result = evaluateDocument([], [], 'fr'); + assert.equal(result.passed, true); + assert.equal(result.chunkCount, 0); + }); + + it('sample size caps samples', () => { + const result = evaluateDocument( + Array(10).fill('hi'), + Array(10).fill('مرحبا'), + 'fr', + 3, + ); + assert.equal(result.failedChunkCount, 10); + assert.equal(result.samples.length, 3); + }); +}); + +// ---------- detectArabicVariant direct ---------- + +describe('detectArabicVariant', () => { + it('persian for persian passes', () => { + const r = detectArabicVariant('سلام چطوری', 'fa'); + assert.equal(r.verdict, 'pass'); + }); + + it('persian for arabic fails', () => { + const r = detectArabicVariant('سلام چطوری', 'ar'); + assert.equal(r.verdict, 'fail'); + assert.ok(r.detectedVariants.includes('fa')); + }); + + it('urdu for persian fails', () => { + const r = detectArabicVariant('السلام ٹڈ', 'fa'); + assert.equal(r.verdict, 'fail'); + assert.ok(r.detectedVariants.includes('ur')); + }); + + it('non-arabic text skipped', () => { + const r = detectArabicVariant('Bonjour le monde', 'fa'); + assert.equal(r.verdict, 'skip'); + }); +}); + +// ---------- lengthCheck direct ---------- + +describe('lengthCheck', () => { + it('normal length returns ratio', () => { + const r = lengthCheck( + 'Hello world this is a longer source string', + 'Bonjour le monde ceci est une traduction francaise', + ); + assert.equal(r.issue, null); + assert.ok(r.ratio !== null); + }); + + it('huge translation', () => { + const r = lengthCheck('A'.repeat(200), 'x'.repeat(1000)); + assert.equal(r.issue, 'length_outlier'); + }); + + it('tiny translation', () => { + const r = lengthCheck('A'.repeat(100), 'ok'); + assert.equal(r.issue, 'truncation_suspect'); + }); + + it('empty translation flagged', () => { + const r = lengthCheck('Hello world this is a test', ''); + assert.equal(r.issue, 'truncation_suspect'); + }); +}); + +// ---------- patternCheck direct ---------- + +describe('patternCheck', () => { + it('no leak on normal text', () => { + const r = patternCheck('Bonjour le monde'); + assert.equal(r.issue, null); + }); + + it('english leak', () => { + const r = patternCheck('Translation: Bonjour'); + assert.equal(r.issue, 'prompt_leak'); + }); + + it('french leak', () => { + const r = patternCheck('Voici la traduction : Bonjour'); + assert.equal(r.issue, 'prompt_leak'); + }); + + it('chinese leak', () => { + const r = patternCheck('翻译：你好'); + assert.equal(r.issue, 'prompt_leak'); + }); + + it('token repetition', () => { + const r = patternCheck('the the the the the the the'); + assert.equal(r.issue, 'repetition_hallucination'); + }); + + it('char repetition', () => { + const r = patternCheck('x'.repeat(30)); + assert.equal(r.issue, 'repetition_hallucination'); + }); +});