sepehr
d92bbf0fa6
Le defaut le plus sournois du pipeline etait silencieux : de l'arabe livre pour une cible persane (meme ecriture, mauvaise langue) passe inapercu et part chez le lecteur. Desormais chaque lot traduit est verifie par le detecteur d'ecritures, et chaque segment fautif est redemande une fois au moteur avec une consigne renforcee (nom de la langue + lettres specifiques, ex. persan پ چ ژ گ). La seconde tentative ne remplace la premiere que si elle passe le meme controle. - services/quality/script_detector.py : extraction d'un controle script_issue() reutilisable ; detect_arabic_variant signale des ormais un long texte en ecriture arabe sans aucune lettre specifique de la langue cible (arabe pur livre pour du persan/ourdou/pachto) - translators/segments.py : retry_wrong_script() + construction de la consigne renforcee, sans jamais faire echouer le travail - Word, Excel, PDF : branchement apres la memoire de traduction et les validations humaines ; le texte inchange (chiffres, noms propres) ne declenche jamais de retentative - 14 tests nouveaux (tests/test_translators/test_script_retry.py)
445 lines
16 KiB
Python
445 lines
16 KiB
Python
"""
|
|
L0 script detector.
|
|
|
|
Verifies that a translated string is actually written in the script expected
|
|
for the target language.
|
|
|
|
This is the first line of defense against the most common translation
|
|
failure mode: the LLM hallucinates text in the wrong language or wrong
|
|
script (e.g. user asks for Persian, model returns Arabic, or user asks
|
|
for Hindi, model returns Arabic). The check is purely heuristic — it
|
|
counts code points in the relevant Unicode ranges and compares to a
|
|
threshold.
|
|
|
|
Pure Python. No network calls. No new dependencies.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field, asdict
|
|
from typing import Dict, List, Optional
|
|
|
|
from . import config as _config
|
|
from . import length_checker
|
|
from . import pattern_leak
|
|
|
|
from core.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
# ---------- Result dataclasses ----------
|
|
|
|
@dataclass
|
|
class QualityCheckResult:
|
|
"""Result of evaluating a single (source, translation) pair."""
|
|
passed: bool
|
|
score: float # 0.0 to 1.0
|
|
issues: List[str] = field(default_factory=list)
|
|
detected_script: Optional[str] = None
|
|
expected_script: Optional[str] = None
|
|
details: Dict = field(default_factory=dict)
|
|
|
|
def to_log_dict(self) -> Dict:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass
|
|
class DocumentQualityResult:
|
|
"""Aggregated result for a list of (source, translation) pairs."""
|
|
passed: bool
|
|
score: float # mean score across chunks
|
|
chunk_count: int
|
|
failed_chunk_count: int
|
|
issues: Dict[str, int] = field(default_factory=dict) # issue -> count
|
|
samples: List[Dict] = field(default_factory=list) # a few example failures
|
|
|
|
def to_log_dict(self) -> Dict:
|
|
return asdict(self)
|
|
|
|
|
|
# ---------- Core helpers ----------
|
|
|
|
def _char_in_ranges(code_point: int, ranges: list) -> bool:
|
|
"""True if a code point falls in any of the (start, end) ranges."""
|
|
for start, end in ranges:
|
|
if start <= code_point <= end:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _count_letters(text: str) -> int:
|
|
"""Count alphabetic characters (using Python's built-in isalpha)."""
|
|
return sum(1 for c in text if c.isalpha())
|
|
|
|
|
|
def _count_in_script(text: str, ranges: list) -> int:
|
|
"""Count how many alphabetic characters fall within the given Unicode ranges."""
|
|
if not ranges:
|
|
# 'latin' or unknown — treat all letters as matching.
|
|
return _count_letters(text)
|
|
return sum(
|
|
1 for c in text
|
|
if c.isalpha() and _char_in_ranges(ord(c), ranges)
|
|
)
|
|
|
|
|
|
# ---------- Arabic-script variant detection ----------
|
|
|
|
def detect_arabic_variant(
|
|
text: str,
|
|
claimed_lang: Optional[str],
|
|
) -> Dict:
|
|
"""
|
|
For text that is in the Arabic script block, check whether it matches
|
|
the specific variant the user asked for (Persian, Urdu, Pashto, etc.).
|
|
|
|
Returns a dict like:
|
|
{
|
|
"verdict": "pass" | "fail" | "skip",
|
|
"claimed_lang": "fa",
|
|
"detected_variants": ["fa"],
|
|
"reason": "...",
|
|
}
|
|
|
|
Detection logic:
|
|
1. If the text has < 60% Arabic-script letters overall, verdict = "skip"
|
|
(the script-detector will catch the mismatch).
|
|
2. If claimed_lang is NOT an Arabic-script language, verdict = "fail"
|
|
(this case should have been caught upstream — defensive double-check).
|
|
3. Scan the text for any discriminating character from any
|
|
Arabic-script language. If a discriminating character of a
|
|
DIFFERENT language is found, verdict = "fail".
|
|
4. Otherwise verdict = "pass".
|
|
"""
|
|
if not text or not text.strip():
|
|
return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "empty text"}
|
|
|
|
arabic_ranges = _config.get_ranges("arabic")
|
|
letters = _count_letters(text)
|
|
if letters == 0:
|
|
return {"verdict": "skip", "claimed_lang": claimed_lang, "reason": "no letters"}
|
|
|
|
in_arabic = _count_in_script(text, arabic_ranges)
|
|
arabic_ratio = in_arabic / letters
|
|
|
|
if arabic_ratio < _config.MIN_RATIO_IN_SCRIPT:
|
|
# Not really Arabic-script — let the main script_detector handle it.
|
|
return {
|
|
"verdict": "skip",
|
|
"claimed_lang": claimed_lang,
|
|
"arabic_ratio": round(arabic_ratio, 3),
|
|
"reason": "not in Arabic script",
|
|
}
|
|
|
|
if not _config.is_arabic_script_lang(claimed_lang):
|
|
# The translation IS in Arabic but the target wasn't Arabic.
|
|
# The main script_detector will fail on this; we just return skip.
|
|
return {
|
|
"verdict": "skip",
|
|
"claimed_lang": claimed_lang,
|
|
"arabic_ratio": round(arabic_ratio, 3),
|
|
"reason": "target is not an Arabic-script language",
|
|
}
|
|
|
|
# Now: text is Arabic-script AND target is Arabic-script. Check the variant.
|
|
detected = set()
|
|
for lang_code, chars in _config.DISCRIMINATING_CHARS.items():
|
|
if not chars:
|
|
continue
|
|
if any(c in chars for c in text):
|
|
detected.add(lang_code)
|
|
|
|
if detected and claimed_lang and claimed_lang.lower() not in detected:
|
|
return {
|
|
"verdict": "fail",
|
|
"claimed_lang": claimed_lang,
|
|
"detected_variants": sorted(detected),
|
|
"arabic_ratio": round(arabic_ratio, 3),
|
|
"reason": (
|
|
f"target={claimed_lang} but text contains characters typical of "
|
|
f"{', '.join(sorted(detected))}"
|
|
),
|
|
}
|
|
|
|
# No discriminating character of ANY Arabic-script variant was found.
|
|
# For a long text this is itself a strong signal: a Persian/Urdu/Pashto
|
|
# text of this length almost always contains at least one of its
|
|
# specific letters (Persian پ چ ژ گ are common letters), while plain
|
|
# Arabic never does. Short texts stay tolerant — the signal is too
|
|
# weak to tell plain Arabic from e.g. Persian without those letters.
|
|
target_chars = _config.get_discriminating_chars(claimed_lang)
|
|
if (
|
|
not detected
|
|
and claimed_lang
|
|
and claimed_lang.lower() != "ar"
|
|
and target_chars
|
|
and letters >= 25
|
|
):
|
|
return {
|
|
"verdict": "fail",
|
|
"claimed_lang": claimed_lang,
|
|
"detected_variants": [],
|
|
"arabic_ratio": round(arabic_ratio, 3),
|
|
"reason": (
|
|
f"target={claimed_lang} but none of its specific letters appear "
|
|
f"in a {letters}-letter Arabic-script text — likely plain Arabic"
|
|
),
|
|
}
|
|
|
|
return {
|
|
"verdict": "pass",
|
|
"claimed_lang": claimed_lang,
|
|
"detected_variants": sorted(detected) if detected else [claimed_lang],
|
|
"arabic_ratio": round(arabic_ratio, 3),
|
|
"reason": "ok",
|
|
}
|
|
|
|
|
|
# ---------- Per-chunk evaluation ----------
|
|
|
|
def _script_checks(text: str, target_lang: Optional[str]) -> tuple:
|
|
"""
|
|
Script checks shared by evaluate_chunk (QA) and script_issue (retry).
|
|
|
|
Returns (issues, details): issues is a list of
|
|
"wrong_script"/"wrong_arabic_variant" strings, details carries the
|
|
diagnostics evaluate_chunk logs.
|
|
"""
|
|
issues: List[str] = []
|
|
details: Dict = {}
|
|
|
|
# --- Script detection ---
|
|
expected_script = _config.get_script(target_lang)
|
|
expected_ranges = _config.get_ranges(expected_script)
|
|
letters = _count_letters(text)
|
|
|
|
if letters == 0:
|
|
# No alphabetic characters — could be numbers, punctuation, or
|
|
# a single non-Latin symbol. Skip script check.
|
|
script_score = 1.0
|
|
detected_script = expected_script
|
|
details["script_check"] = "skipped: no alphabetic characters"
|
|
else:
|
|
# Always try to determine the ACTUAL script of the text — used for
|
|
# diagnostics and for catching language confusion when the target
|
|
# is Latin (e.g. user asks fr, we get Arabic text).
|
|
detected_script = _detect_actual_script(text)
|
|
in_expected = _count_in_script(text, expected_ranges)
|
|
script_score = in_expected / letters
|
|
|
|
details["script_score"] = round(script_score, 3)
|
|
details["letters_in_text"] = letters
|
|
details["letters_in_script"] = in_expected
|
|
details["detected_script"] = detected_script
|
|
details["expected_script"] = expected_script
|
|
details["min_ratio"] = _config.MIN_RATIO_IN_SCRIPT
|
|
|
|
# Two failure modes:
|
|
# 1. Target is a SPECIFIC non-Latin script (cyrillic, arabic, cjk...)
|
|
# and the text doesn't match it enough.
|
|
# 2. Target is Latin but the text is clearly in a SPECIFIC other
|
|
# script (cyrillic, arabic, devanagari, cjk...) — language
|
|
# confusion.
|
|
if expected_script != "latin" and expected_ranges:
|
|
# Specific non-Latin target.
|
|
if script_score < _config.MIN_RATIO_IN_SCRIPT:
|
|
issues.append("wrong_script")
|
|
details["reason"] = (
|
|
f"only {script_score:.0%} of letters match {expected_script} script; "
|
|
f"text appears to be in {detected_script}"
|
|
)
|
|
else:
|
|
# Latin target. If detected script is clearly non-Latin, fail.
|
|
if detected_script and detected_script != "latin" and detected_script != "unknown":
|
|
# Measure how confident we are that the text is non-Latin.
|
|
non_latin_ranges = _config.get_ranges(detected_script)
|
|
in_detected = _count_in_script(text, non_latin_ranges)
|
|
non_latin_confidence = in_detected / letters
|
|
if non_latin_confidence >= 0.7:
|
|
issues.append("wrong_script")
|
|
details["reason"] = (
|
|
f"target is Latin but {non_latin_confidence:.0%} of letters "
|
|
f"are in {detected_script} script — language confusion"
|
|
)
|
|
|
|
# --- Arabic-script variant detection ---
|
|
if _config.is_arabic_script_lang(target_lang):
|
|
variant_result = detect_arabic_variant(text, target_lang)
|
|
details["arabic_variant"] = variant_result
|
|
if variant_result["verdict"] == "fail":
|
|
issues.append("wrong_arabic_variant")
|
|
|
|
return issues, details
|
|
|
|
|
|
def script_issue(translated_text: str, target_lang: Optional[str]) -> Optional[str]:
|
|
"""
|
|
Standalone wrong-script check, usable by the translators to decide
|
|
whether a delivered translation must be retried.
|
|
|
|
Returns a short human-readable reason when the text is NOT written
|
|
in the script expected for target_lang (wrong script, or wrong
|
|
Arabic-script variant such as Arabic delivered for Persian), and
|
|
None when the script is acceptable. Purely heuristic, never raises.
|
|
"""
|
|
try:
|
|
text = (translated_text or "").strip()
|
|
if not text:
|
|
return None
|
|
issues, details = _script_checks(text, (target_lang or "").lower() or None)
|
|
if not issues:
|
|
return None
|
|
if "wrong_arabic_variant" in issues:
|
|
variant = details.get("arabic_variant") or {}
|
|
return variant.get("reason") or "wrong_arabic_variant"
|
|
return details.get("reason") or issues[0]
|
|
except Exception as e: # defensive: retry decisions must never crash a job
|
|
logger.warning("script_issue_error", error=str(e)[:200])
|
|
return None
|
|
|
|
|
|
def evaluate_chunk(
|
|
source_text: str,
|
|
translated_text: str,
|
|
target_lang: Optional[str],
|
|
) -> QualityCheckResult:
|
|
"""
|
|
Run the L0 checks on a single (source, translation) pair.
|
|
|
|
Returns a QualityCheckResult. The function is purely defensive — it
|
|
never raises; any internal error results in a "skip" result.
|
|
"""
|
|
if translated_text is None:
|
|
return QualityCheckResult(
|
|
passed=True, score=0.0, issues=["empty_translation"],
|
|
details={"reason": "translation is None"},
|
|
)
|
|
|
|
text = translated_text.strip()
|
|
if not text:
|
|
return QualityCheckResult(
|
|
passed=True, score=0.0, issues=["empty_translation"],
|
|
details={"reason": "translation is empty or whitespace-only"},
|
|
)
|
|
|
|
target_lang = (target_lang or "").lower() or None
|
|
issues: List[str] = []
|
|
details: Dict = {}
|
|
|
|
# --- Script detection + Arabic variant (shared with script_issue) ---
|
|
script_issues, script_details = _script_checks(text, target_lang)
|
|
issues.extend(script_issues)
|
|
details.update(script_details)
|
|
|
|
# --- Length sanity ---
|
|
length_result = length_checker.check(source_text, text)
|
|
details["length"] = length_result
|
|
if length_result.get("issue"):
|
|
issues.append(length_result["issue"])
|
|
|
|
# --- Pattern leak / repetition ---
|
|
leak_result = pattern_leak.check(text)
|
|
details["pattern_check"] = leak_result
|
|
if leak_result.get("issue"):
|
|
issues.append(leak_result["issue"])
|
|
|
|
# --- Aggregate ---
|
|
passed = len(issues) == 0
|
|
# Simple score: how many of the 3 main checks passed.
|
|
n_checks = 3
|
|
n_failed = sum(
|
|
1 for issue in issues if issue in (
|
|
"wrong_script", "wrong_arabic_variant",
|
|
"length_outlier", "truncation_suspect",
|
|
"prompt_leak", "repetition_hallucination",
|
|
)
|
|
)
|
|
score = max(0.0, 1.0 - (n_failed / n_checks))
|
|
|
|
return QualityCheckResult(
|
|
passed=passed,
|
|
score=round(score, 3),
|
|
issues=issues,
|
|
detected_script=script_details.get("detected_script"),
|
|
expected_script=script_details.get("expected_script"),
|
|
details=details,
|
|
)
|
|
|
|
|
|
def _detect_actual_script(text: str) -> str:
|
|
"""
|
|
Heuristically determine which script a string is in. Used only for
|
|
diagnostics — never for the verdict. Returns the first script (in
|
|
priority order) whose ratio exceeds the threshold.
|
|
"""
|
|
letters = _count_letters(text)
|
|
if letters == 0:
|
|
return "unknown"
|
|
# Priority order: more specific scripts first.
|
|
order = [
|
|
"hiragana_katakana", "hangul", "cjk", "thai", "lao", "burmese",
|
|
"khmer", "devanagari", "bengali", "tamil", "telugu", "kannada",
|
|
"malayalam", "sinhala", "gujarati", "gurmukhi",
|
|
"arabic", "hebrew", "cyrillic", "greek", "armenian", "georgian",
|
|
"ethiopic", "tibetan", "thaana",
|
|
]
|
|
for script_id in order:
|
|
ranges = _config.get_ranges(script_id)
|
|
in_script = _count_in_script(text, ranges)
|
|
if in_script / letters > 0.4:
|
|
return script_id
|
|
return "latin"
|
|
|
|
|
|
# ---------- Document-level aggregation ----------
|
|
|
|
def evaluate_document(
|
|
source_chunks: List[str],
|
|
translated_chunks: List[str],
|
|
target_lang: Optional[str],
|
|
sample_size: int = 50,
|
|
) -> DocumentQualityResult:
|
|
"""
|
|
Evaluate all (source, translation) pairs and return a document-level
|
|
summary. The full list is processed but only the first `sample_size`
|
|
failing chunks are kept in `samples` to keep logs compact.
|
|
"""
|
|
n = min(len(source_chunks), len(translated_chunks))
|
|
chunk_results: List[QualityCheckResult] = []
|
|
issues_count: Dict[str, int] = {}
|
|
samples: List[Dict] = []
|
|
score_sum = 0.0
|
|
failed_count = 0
|
|
|
|
for i in range(n):
|
|
r = evaluate_chunk(source_chunks[i], translated_chunks[i], target_lang)
|
|
chunk_results.append(r)
|
|
score_sum += r.score
|
|
for issue in r.issues:
|
|
issues_count[issue] = issues_count.get(issue, 0) + 1
|
|
if not r.passed:
|
|
failed_count += 1
|
|
if len(samples) < sample_size:
|
|
src_preview = (source_chunks[i] or "")[:80]
|
|
trans_preview = (translated_chunks[i] or "")[:80]
|
|
samples.append({
|
|
"index": i,
|
|
"issues": r.issues,
|
|
"source_preview": src_preview,
|
|
"translated_preview": trans_preview,
|
|
"details": r.details,
|
|
})
|
|
|
|
mean_score = (score_sum / n) if n > 0 else 0.0
|
|
passed = failed_count == 0
|
|
|
|
return DocumentQualityResult(
|
|
passed=passed,
|
|
score=round(mean_score, 3),
|
|
chunk_count=n,
|
|
failed_chunk_count=failed_count,
|
|
issues=issues_count,
|
|
samples=samples,
|
|
)
|