feat(qualite): retenter automatiquement les traductions livrees dans la mauvaise ecriture
Le defaut le plus sournois du pipeline etait silencieux : de l'arabe livre pour une cible persane (meme ecriture, mauvaise langue) passe inapercu et part chez le lecteur. Desormais chaque lot traduit est verifie par le detecteur d'ecritures, et chaque segment fautif est redemande une fois au moteur avec une consigne renforcee (nom de la langue + lettres specifiques, ex. persan پ چ ژ گ). La seconde tentative ne remplace la premiere que si elle passe le meme controle. - services/quality/script_detector.py : extraction d'un controle script_issue() reutilisable ; detect_arabic_variant signale des ormais un long texte en ecriture arabe sans aucune lettre specifique de la langue cible (arabe pur livre pour du persan/ourdou/pachto) - translators/segments.py : retry_wrong_script() + construction de la consigne renforcee, sans jamais faire echouer le travail - Word, Excel, PDF : branchement apres la memoire de traduction et les validations humaines ; le texte inchange (chiffres, noms propres) ne declenche jamais de retentative - 14 tests nouveaux (tests/test_translators/test_script_retry.py)
This commit is contained in:
@@ -162,6 +162,31 @@ def detect_arabic_variant(
|
||||
),
|
||||
}
|
||||
|
||||
# No discriminating character of ANY Arabic-script variant was found.
|
||||
# For a long text this is itself a strong signal: a Persian/Urdu/Pashto
|
||||
# text of this length almost always contains at least one of its
|
||||
# specific letters (Persian پ چ ژ گ are common letters), while plain
|
||||
# Arabic never does. Short texts stay tolerant — the signal is too
|
||||
# weak to tell plain Arabic from e.g. Persian without those letters.
|
||||
target_chars = _config.get_discriminating_chars(claimed_lang)
|
||||
if (
|
||||
not detected
|
||||
and claimed_lang
|
||||
and claimed_lang.lower() != "ar"
|
||||
and target_chars
|
||||
and letters >= 25
|
||||
):
|
||||
return {
|
||||
"verdict": "fail",
|
||||
"claimed_lang": claimed_lang,
|
||||
"detected_variants": [],
|
||||
"arabic_ratio": round(arabic_ratio, 3),
|
||||
"reason": (
|
||||
f"target={claimed_lang} but none of its specific letters appear "
|
||||
f"in a {letters}-letter Arabic-script text — likely plain Arabic"
|
||||
),
|
||||
}
|
||||
|
||||
return {
|
||||
"verdict": "pass",
|
||||
"claimed_lang": claimed_lang,
|
||||
@@ -173,31 +198,14 @@ def detect_arabic_variant(
|
||||
|
||||
# ---------- Per-chunk evaluation ----------
|
||||
|
||||
def evaluate_chunk(
|
||||
source_text: str,
|
||||
translated_text: str,
|
||||
target_lang: Optional[str],
|
||||
) -> QualityCheckResult:
|
||||
def _script_checks(text: str, target_lang: Optional[str]) -> tuple:
|
||||
"""
|
||||
Run the L0 checks on a single (source, translation) pair.
|
||||
Script checks shared by evaluate_chunk (QA) and script_issue (retry).
|
||||
|
||||
Returns a QualityCheckResult. The function is purely defensive — it
|
||||
never raises; any internal error results in a "skip" result.
|
||||
Returns (issues, details): issues is a list of
|
||||
"wrong_script"/"wrong_arabic_variant" strings, details carries the
|
||||
diagnostics evaluate_chunk logs.
|
||||
"""
|
||||
if translated_text is None:
|
||||
return QualityCheckResult(
|
||||
passed=True, score=0.0, issues=["empty_translation"],
|
||||
details={"reason": "translation is None"},
|
||||
)
|
||||
|
||||
text = translated_text.strip()
|
||||
if not text:
|
||||
return QualityCheckResult(
|
||||
passed=True, score=0.0, issues=["empty_translation"],
|
||||
details={"reason": "translation is empty or whitespace-only"},
|
||||
)
|
||||
|
||||
target_lang = (target_lang or "").lower() or None
|
||||
issues: List[str] = []
|
||||
details: Dict = {}
|
||||
|
||||
@@ -262,6 +270,68 @@ def evaluate_chunk(
|
||||
if variant_result["verdict"] == "fail":
|
||||
issues.append("wrong_arabic_variant")
|
||||
|
||||
return issues, details
|
||||
|
||||
|
||||
def script_issue(translated_text: str, target_lang: Optional[str]) -> Optional[str]:
|
||||
"""
|
||||
Standalone wrong-script check, usable by the translators to decide
|
||||
whether a delivered translation must be retried.
|
||||
|
||||
Returns a short human-readable reason when the text is NOT written
|
||||
in the script expected for target_lang (wrong script, or wrong
|
||||
Arabic-script variant such as Arabic delivered for Persian), and
|
||||
None when the script is acceptable. Purely heuristic, never raises.
|
||||
"""
|
||||
try:
|
||||
text = (translated_text or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
issues, details = _script_checks(text, (target_lang or "").lower() or None)
|
||||
if not issues:
|
||||
return None
|
||||
if "wrong_arabic_variant" in issues:
|
||||
variant = details.get("arabic_variant") or {}
|
||||
return variant.get("reason") or "wrong_arabic_variant"
|
||||
return details.get("reason") or issues[0]
|
||||
except Exception as e: # defensive: retry decisions must never crash a job
|
||||
logger.warning("script_issue_error", error=str(e)[:200])
|
||||
return None
|
||||
|
||||
|
||||
def evaluate_chunk(
|
||||
source_text: str,
|
||||
translated_text: str,
|
||||
target_lang: Optional[str],
|
||||
) -> QualityCheckResult:
|
||||
"""
|
||||
Run the L0 checks on a single (source, translation) pair.
|
||||
|
||||
Returns a QualityCheckResult. The function is purely defensive — it
|
||||
never raises; any internal error results in a "skip" result.
|
||||
"""
|
||||
if translated_text is None:
|
||||
return QualityCheckResult(
|
||||
passed=True, score=0.0, issues=["empty_translation"],
|
||||
details={"reason": "translation is None"},
|
||||
)
|
||||
|
||||
text = translated_text.strip()
|
||||
if not text:
|
||||
return QualityCheckResult(
|
||||
passed=True, score=0.0, issues=["empty_translation"],
|
||||
details={"reason": "translation is empty or whitespace-only"},
|
||||
)
|
||||
|
||||
target_lang = (target_lang or "").lower() or None
|
||||
issues: List[str] = []
|
||||
details: Dict = {}
|
||||
|
||||
# --- Script detection + Arabic variant (shared with script_issue) ---
|
||||
script_issues, script_details = _script_checks(text, target_lang)
|
||||
issues.extend(script_issues)
|
||||
details.update(script_details)
|
||||
|
||||
# --- Length sanity ---
|
||||
length_result = length_checker.check(source_text, text)
|
||||
details["length"] = length_result
|
||||
@@ -291,8 +361,8 @@ def evaluate_chunk(
|
||||
passed=passed,
|
||||
score=round(score, 3),
|
||||
issues=issues,
|
||||
detected_script=detected_script,
|
||||
expected_script=expected_script,
|
||||
detected_script=script_details.get("detected_script"),
|
||||
expected_script=script_details.get("expected_script"),
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user