""" L0 script detector. Verifies that a translated string is actually written in the script expected for the target language. This is the first line of defense against the most common translation failure mode: the LLM hallucinates text in the wrong language or wrong script (e.g. user asks for Persian, model returns Arabic, or user asks for Hindi, model returns Arabic). The check is purely heuristic — it counts code points in the relevant Unicode ranges and compares to a threshold. Pure Python. No network calls. No new dependencies. """ from __future__ import annotations from dataclasses import dataclass, field, asdict from typing import Dict, List, Optional from . import config as _config from . import length_checker from . import pattern_leak from core.logging import get_logger logger = get_logger(__name__) # ---------- Result dataclasses ---------- @dataclass class QualityCheckResult: """Result of evaluating a single (source, translation) pair.""" passed: bool score: float # 0.0 to 1.0 issues: List[str] = field(default_factory=list) detected_script: Optional[str] = None expected_script: Optional[str] = None details: Dict = field(default_factory=dict) def to_log_dict(self) -> Dict: return asdict(self) @dataclass class DocumentQualityResult: """Aggregated result for a list of (source, translation) pairs.""" passed: bool score: float # mean score across chunks chunk_count: int failed_chunk_count: int issues: Dict[str, int] = field(default_factory=dict) # issue -> count samples: List[Dict] = field(default_factory=list) # a few example failures def to_log_dict(self) -> Dict: return asdict(self) # ---------- Core helpers ---------- def _char_in_ranges(code_point: int, ranges: list) -> bool: """True if a code point falls in any of the (start, end) ranges.""" for start, end in ranges: if start <= code_point <= end: return True return False def _count_letters(text: str) -> int: """Count alphabetic characters (using Python's built-in isalpha).""" return sum(1 for c in text if c.isalpha()) def _count_in_script(text: str, ranges: list) -> int: """Count how many alphabetic characters fall within the given Unicode ranges.""" if not ranges: # 'latin' or unknown — treat all letters as matching. return _count_letters(text) return sum( 1 for c in text if c.isalpha() and _char_in_ranges(ord(c), ranges) ) # ---------- Arabic-script variant detection ---------- def detect_arabic_variant( text: str, claimed_lang: Optional[str], ) -> Dict: """ For text that is in the Arabic script block, check whether it matches the specific variant the user asked for (Persian, Urdu, Pashto, etc.). Returns a dict like: { "verdict": "pass" | "fail" | "skip", "claimed_lang": "fa", "detected_variants": ["fa"], "reason": "...", } Detection logic: 1. If the text has < 60% Arabic-script letters overall, verdict = "skip" (the script-detector will catch the mismatch). 2. If claimed_lang is NOT an Arabic-script language, verdict = "fail" (this case should have been caught upstream — defensive double-check). 3. Scan the text for any discriminating character from any Arabic-script language. If a discriminating character of a DIFFERENT language is found, verdict = "fail". 4. Otherwise verdict = "pass". """ if not text or not text.strip(): return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "empty text"} arabic_ranges = _config.get_ranges("arabic") letters = _count_letters(text) if letters == 0: return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "no letters"} in_arabic = _count_in_script(text, arabic_ranges) arabic_ratio = in_arabic / letters if arabic_ratio < _config.MIN_RATIO_IN_SCRIPT: # Not really Arabic-script — let the main script_detector handle it. return { "verdict": "skip", "claimed_lang": claimed_lang, "arabic_ratio": round(arabic_ratio, 3), "reason": "not in Arabic script", } if not _config.is_arabic_script_lang(claimed_lang): # The translation IS in Arabic but the target wasn't Arabic. # The main script_detector will fail on this; we just return skip. return { "verdict": "skip", "claimed_lang": claimed_lang, "arabic_ratio": round(arabic_ratio, 3), "reason": "target is not an Arabic-script language", } # Now: text is Arabic-script AND target is Arabic-script. Check the variant. detected = set() for lang_code, chars in _config.DISCRIMINATING_CHARS.items(): if not chars: continue if any(c in chars for c in text): detected.add(lang_code) if detected and claimed_lang and claimed_lang.lower() not in detected: return { "verdict": "fail", "claimed_lang": claimed_lang, "detected_variants": sorted(detected), "arabic_ratio": round(arabic_ratio, 3), "reason": ( f"target={claimed_lang} but text contains characters typical of " f"{', '.join(sorted(detected))}" ), } return { "verdict": "pass", "claimed_lang": claimed_lang, "detected_variants": sorted(detected) if detected else [claimed_lang], "arabic_ratio": round(arabic_ratio, 3), "reason": "ok", } # ---------- Per-chunk evaluation ---------- def evaluate_chunk( source_text: str, translated_text: str, target_lang: Optional[str], ) -> QualityCheckResult: """ Run the L0 checks on a single (source, translation) pair. Returns a QualityCheckResult. The function is purely defensive — it never raises; any internal error results in a "skip" result. """ if translated_text is None: return QualityCheckResult( passed=True, score=0.0, issues=["empty_translation"], details={"reason": "translation is None"}, ) text = translated_text.strip() if not text: return QualityCheckResult( passed=True, score=0.0, issues=["empty_translation"], details={"reason": "translation is empty or whitespace-only"}, ) target_lang = (target_lang or "").lower() or None issues: List[str] = [] details: Dict = {} # --- Script detection --- expected_script = _config.get_script(target_lang) expected_ranges = _config.get_ranges(expected_script) letters = _count_letters(text) if letters == 0: # No alphabetic characters — could be numbers, punctuation, or # a single non-Latin symbol. Skip script check. script_score = 1.0 detected_script = expected_script details["script_check"] = "skipped: no alphabetic characters" else: # Always try to determine the ACTUAL script of the text — used for # diagnostics and for catching language confusion when the target # is Latin (e.g. user asks fr, we get Arabic text). detected_script = _detect_actual_script(text) in_expected = _count_in_script(text, expected_ranges) script_score = in_expected / letters details["script_score"] = round(script_score, 3) details["letters_in_text"] = letters details["letters_in_script"] = in_expected details["detected_script"] = detected_script details["expected_script"] = expected_script details["min_ratio"] = _config.MIN_RATIO_IN_SCRIPT # Two failure modes: # 1. Target is a SPECIFIC non-Latin script (cyrillic, arabic, cjk...) # and the text doesn't match it enough. # 2. Target is Latin but the text is clearly in a SPECIFIC other # script (cyrillic, arabic, devanagari, cjk...) — language # confusion. if expected_script != "latin" and expected_ranges: # Specific non-Latin target. if script_score < _config.MIN_RATIO_IN_SCRIPT: issues.append("wrong_script") details["reason"] = ( f"only {script_score:.0%} of letters match {expected_script} script; " f"text appears to be in {detected_script}" ) else: # Latin target. If detected script is clearly non-Latin, fail. if detected_script and detected_script != "latin" and detected_script != "unknown": # Measure how confident we are that the text is non-Latin. non_latin_ranges = _config.get_ranges(detected_script) in_detected = _count_in_script(text, non_latin_ranges) non_latin_confidence = in_detected / letters if non_latin_confidence >= 0.7: issues.append("wrong_script") details["reason"] = ( f"target is Latin but {non_latin_confidence:.0%} of letters " f"are in {detected_script} script — language confusion" ) # --- Arabic-script variant detection --- if _config.is_arabic_script_lang(target_lang): variant_result = detect_arabic_variant(text, target_lang) details["arabic_variant"] = variant_result if variant_result["verdict"] == "fail": issues.append("wrong_arabic_variant") # --- Length sanity --- length_result = length_checker.check(source_text, text) details["length"] = length_result if length_result.get("issue"): issues.append(length_result["issue"]) # --- Pattern leak / repetition --- leak_result = pattern_leak.check(text) details["pattern_check"] = leak_result if leak_result.get("issue"): issues.append(leak_result["issue"]) # --- Aggregate --- passed = len(issues) == 0 # Simple score: how many of the 3 main checks passed. n_checks = 3 n_failed = sum( 1 for issue in issues if issue in ( "wrong_script", "wrong_arabic_variant", "length_outlier", "truncation_suspect", "prompt_leak", "repetition_hallucination", ) ) score = max(0.0, 1.0 - (n_failed / n_checks)) return QualityCheckResult( passed=passed, score=round(score, 3), issues=issues, detected_script=detected_script, expected_script=expected_script, details=details, ) def _detect_actual_script(text: str) -> str: """ Heuristically determine which script a string is in. Used only for diagnostics — never for the verdict. Returns the first script (in priority order) whose ratio exceeds the threshold. """ letters = _count_letters(text) if letters == 0: return "unknown" # Priority order: more specific scripts first. order = [ "hiragana_katakana", "hangul", "cjk", "thai", "lao", "burmese", "khmer", "devanagari", "bengali", "tamil", "telugu", "kannada", "malayalam", "sinhala", "gujarati", "gurmukhi", "arabic", "hebrew", "cyrillic", "greek", "armenian", "georgian", "ethiopic", "tibetan", "thaana", ] for script_id in order: ranges = _config.get_ranges(script_id) in_script = _count_in_script(text, ranges) if in_script / letters > 0.4: return script_id return "latin" # ---------- Document-level aggregation ---------- def evaluate_document( source_chunks: List[str], translated_chunks: List[str], target_lang: Optional[str], sample_size: int = 50, ) -> DocumentQualityResult: """ Evaluate all (source, translation) pairs and return a document-level summary. The full list is processed but only the first `sample_size` failing chunks are kept in `samples` to keep logs compact. """ n = min(len(source_chunks), len(translated_chunks)) chunk_results: List[QualityCheckResult] = [] issues_count: Dict[str, int] = {} samples: List[Dict] = [] score_sum = 0.0 failed_count = 0 for i in range(n): r = evaluate_chunk(source_chunks[i], translated_chunks[i], target_lang) chunk_results.append(r) score_sum += r.score for issue in r.issues: issues_count[issue] = issues_count.get(issue, 0) + 1 if not r.passed: failed_count += 1 if len(samples) < sample_size: src_preview = (source_chunks[i] or "")[:80] trans_preview = (translated_chunks[i] or "")[:80] samples.append({ "index": i, "issues": r.issues, "source_preview": src_preview, "translated_preview": trans_preview, "details": r.details, }) mean_score = (score_sum / n) if n > 0 else 0.0 passed = failed_count == 0 return DocumentQualityResult( passed=passed, score=round(mean_score, 3), chunk_count=n, failed_chunk_count=failed_count, issues=issues_count, samples=samples, )