feat(quality): add L0 quality layer (Track A1 + A2 of dev plan)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
L0 quality detection layer to catch translation failures BEFORE they
reach users. Pure Python/TypeScript, zero new dependencies, no API calls.
Backend (Python — services/quality/):
- Script detection: 145 langs mapped to 23 scripts (Latin, Cyrillic,
Greek, Arabic, Hebrew, CJK, Hangul, Kana, Devanagari, Bengali, etc.)
- Language confusion detection (e.g. Arabic text for French target)
- Arabic-script variant discrimination (Persian/Urdu/Pashto/Kurdish
confusion — e.g. Persian text returned when Arabic was requested)
- Length sanity check (with numeric/short-source exemptions)
- Prompt leak detection (Translation: / Voici la traduction: / 翻译:)
- Repetition hallucination detection (token + character level)
- File text extraction for .docx/.xlsx/.pptx/.pdf (no translator
changes needed)
- Defensive pipeline that never raises (L0 must NEVER break a job)
Frontend (TypeScript — wordly.art---traduction-de-documents/src/utils/):
- Exact 1:1 mirror of the Python module
- Zero dependencies, works in browser AND Node.js
- Native Unicode regex (\\p{L}/u) and codePoint iteration
- 63 tests using Node's built-in test runner
Integration:
- Feature-flagged: QUALITY_L0_ENABLED=false (default)
- Observation only: logs structured events, never modifies files
- try/except wrapped: impossible to break a translation job
- Lazy imports: only loaded when flag is on
- Zero impact on existing tests / behavior
Tests:
- 111 Python tests covering all paths (config, script, length, leak,
pipeline, file_extractor) — 100% pass
- 63 TypeScript tests (Node --test) — 100% pass
- 174/174 total tests for the L0 layer
Bug fixes in script mapping:
- yi (Yiddish) -> hebrew (was incorrectly mapped to arabic)
- dv (Maldivian) -> thaana (was incorrectly mapped to arabic)
- ja (Japanese) -> hiragana_katakana (distinguishes from Chinese CJK)
Phase 1 (backend) + Phase 2 (frontend) of Track A complete.
Next: Track B1 (Word/Excel format preservation quick wins).
Closes Track A phase 1+2 of the dev plan.
This commit is contained in:
290
tests/services/quality/test_script_detector.py
Normal file
290
tests/services/quality/test_script_detector.py
Normal file
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Tests for services/quality/script_detector.py
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality import evaluate_chunk, evaluate_document, detect_arabic_variant
|
||||
|
||||
|
||||
# ---------- Happy path: correct script ----------
|
||||
|
||||
class TestCorrectScript:
|
||||
def test_russian_correct(self):
|
||||
r = evaluate_chunk("Hello world", "Привет мир", "ru")
|
||||
assert r.passed is True
|
||||
assert "wrong_script" not in r.issues
|
||||
|
||||
def test_chinese_simplified_correct(self):
|
||||
r = evaluate_chunk("Hello", "你好世界", "zh")
|
||||
assert r.passed is True
|
||||
|
||||
def test_arabic_correct(self):
|
||||
r = evaluate_chunk("Hello", "مرحبا بالعالم", "ar")
|
||||
assert r.passed is True
|
||||
|
||||
def test_persian_correct(self):
|
||||
# Persian has unique chars چ پ ژ گ
|
||||
r = evaluate_chunk("Hello", "سلام چطوری؟ من پژوهشگر هستم", "fa")
|
||||
assert r.passed is True
|
||||
|
||||
def test_french_correct(self):
|
||||
r = evaluate_chunk("Hello world", "Bonjour le monde", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_hebrew_correct(self):
|
||||
r = evaluate_chunk("Hello", "שלום עולם", "he")
|
||||
assert r.passed is True
|
||||
|
||||
def test_korean_correct(self):
|
||||
r = evaluate_chunk("Hello", "안녕하세요 세계", "ko")
|
||||
assert r.passed is True
|
||||
|
||||
def test_japanese_correct(self):
|
||||
# Japanese must contain hiragana or katakana to be classified as ja.
|
||||
r = evaluate_chunk("Hello", "こんにちは世界", "ja")
|
||||
assert r.passed is True
|
||||
|
||||
def test_hindi_correct(self):
|
||||
r = evaluate_chunk("Hello", "नमस्ते दुनिया", "hi")
|
||||
assert r.passed is True
|
||||
|
||||
def test_thai_correct(self):
|
||||
r = evaluate_chunk("Hello", "สวัสดีชาวโลก", "th")
|
||||
assert r.passed is True
|
||||
|
||||
def test_greek_correct(self):
|
||||
r = evaluate_chunk("Hello", "Γεια σας κόσμε", "el")
|
||||
assert r.passed is True
|
||||
|
||||
|
||||
# ---------- Wrong script: language confusion ----------
|
||||
|
||||
class TestWrongScript:
|
||||
def test_french_text_for_japanese_target(self):
|
||||
r = evaluate_chunk("Hello", "Bonjour le monde", "ja")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_arabic_text_for_french_target(self):
|
||||
r = evaluate_chunk("Hello", "مرحبا بالعالم", "fr")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_russian_text_for_english_target(self):
|
||||
r = evaluate_chunk("Hello", "Привет мир", "en")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_chinese_text_for_korean_target(self):
|
||||
r = evaluate_chunk("Hello", "你好世界", "ko")
|
||||
# Hangul syllables are 0xAC00-0xD7AF — Chinese chars are not in that range.
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
|
||||
# ---------- Persian / Arabic discrimination ----------
|
||||
|
||||
class TestArabicVariantDiscrimination:
|
||||
def test_persian_for_arabic_target_fails(self):
|
||||
# Persian text with چ پ ژ گ should fail when target is Arabic.
|
||||
r = evaluate_chunk("Hello", "سلام چطوری؟", "ar")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
def test_arabic_for_persian_target_fails(self):
|
||||
# Pure Arabic text (no Persian-specific chars) when target is Persian.
|
||||
r = evaluate_chunk("Hello", "السلام عليكم", "fa")
|
||||
# No Persian-specific chars, no other Arabic-script variant chars
|
||||
# → should pass the variant check (because no discrimination signal).
|
||||
# This is a known limitation: pure Arabic indistinguishable from pure Persian.
|
||||
# We accept that the variant check only flags MISMATCHES with discriminating chars.
|
||||
assert "wrong_arabic_variant" not in r.issues
|
||||
|
||||
def test_urdu_for_persian_target_fails(self):
|
||||
# Urdu with ٹ ڈ should fail when target is Persian.
|
||||
r = evaluate_chunk("Hello", "السلام ٹڈ", "fa")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
def test_pashto_for_arabic_target_fails(self):
|
||||
# Pashto with ټډړ should fail when target is Arabic.
|
||||
r = evaluate_chunk("Hello", "السلام ټډړ", "ar")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
|
||||
# ---------- Edge cases ----------
|
||||
|
||||
class TestEdgeCases:
|
||||
def test_empty_translation_passes(self):
|
||||
# Empty translations are skipped (the provider may have legitimately
|
||||
# produced nothing — e.g. an empty cell).
|
||||
r = evaluate_chunk("Hello", "", "fr")
|
||||
assert r.passed is True
|
||||
assert "empty_translation" in r.issues
|
||||
|
||||
def test_whitespace_only_passes(self):
|
||||
r = evaluate_chunk("Hello", " \n ", "fr")
|
||||
assert r.passed is True
|
||||
assert "empty_translation" in r.issues
|
||||
|
||||
def test_none_translation_passes(self):
|
||||
r = evaluate_chunk("Hello", None, "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_numbers_only_passes(self):
|
||||
r = evaluate_chunk("Price: 100", "100", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_unknown_target_lang_falls_back(self):
|
||||
# Unknown lang → latin → no script check, just length/leak checks.
|
||||
r = evaluate_chunk("Hello", "Bonjour", "xx")
|
||||
assert r.passed is True
|
||||
|
||||
def test_mixed_brands_latin_target(self):
|
||||
# "iPhone 15 Pro Max" is mostly Latin + digits — fine for fr.
|
||||
r = evaluate_chunk("Buy iPhone 15", "Acheter iPhone 15", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
|
||||
# ---------- Length sanity ----------
|
||||
|
||||
class TestLengthIssues:
|
||||
def test_huge_translation_flagged_via_repetition(self):
|
||||
# Source too short for length ratio, but the "x"*1000 repetition
|
||||
# is caught by the pattern/repetition check, not length.
|
||||
r = evaluate_chunk("Short", "x" * 1000, "fr")
|
||||
assert "repetition_hallucination" in r.issues
|
||||
|
||||
def test_huge_translation_with_long_source_flagged(self):
|
||||
# Long enough source → length ratio kicks in → length_outlier.
|
||||
src = "A" * 200
|
||||
r = evaluate_chunk(src, "x" * 1000, "fr")
|
||||
assert "length_outlier" in r.issues
|
||||
|
||||
def test_tiny_translation_flagged(self):
|
||||
r = evaluate_chunk("A" * 100, "ok", "fr")
|
||||
# ratio is 2/100 = 0.02, well below 0.15
|
||||
assert "truncation_suspect" in r.issues
|
||||
|
||||
def test_normal_translation_no_length_issue(self):
|
||||
r = evaluate_chunk("Hello world, how are you?", "Bonjour le monde, comment allez-vous?", "fr")
|
||||
assert "length_outlier" not in r.issues
|
||||
assert "truncation_suspect" not in r.issues
|
||||
|
||||
def test_short_source_skips_ratio(self):
|
||||
r = evaluate_chunk("Hi", "ok", "fr")
|
||||
# source too short for ratio check
|
||||
assert "length_outlier" not in r.issues
|
||||
assert "truncation_suspect" not in r.issues
|
||||
|
||||
def test_numeric_source_skips_length_check(self):
|
||||
# Source is mostly digits — translation can also be short.
|
||||
r = evaluate_chunk("Price: 100", "100", "fr")
|
||||
assert "truncation_suspect" not in r.issues
|
||||
assert "length_outlier" not in r.issues
|
||||
|
||||
|
||||
# ---------- Pattern leak / repetition ----------
|
||||
|
||||
class TestPatternLeak:
|
||||
def test_prompt_leak_english_detected(self):
|
||||
r = evaluate_chunk("Hello", "Translation: Bonjour le monde", "fr")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_prompt_leak_french_detected(self):
|
||||
r = evaluate_chunk("Hello", "Voici la traduction : Bonjour", "fr")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_prompt_leak_chinese_detected(self):
|
||||
r = evaluate_chunk("Hello", "翻译:你好", "zh")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_repetition_hallucination_detected(self):
|
||||
r = evaluate_chunk("Hello", "the the the the the the", "fr")
|
||||
assert "repetition_hallucination" in r.issues
|
||||
|
||||
def test_normal_text_no_leak(self):
|
||||
r = evaluate_chunk("Hello", "Bonjour le monde", "fr")
|
||||
assert "prompt_leak" not in r.issues
|
||||
assert "repetition_hallucination" not in r.issues
|
||||
|
||||
|
||||
# ---------- Document-level aggregation ----------
|
||||
|
||||
class TestEvaluateDocument:
|
||||
def test_all_good(self):
|
||||
result = evaluate_document(
|
||||
["Hello", "World", "Good morning"],
|
||||
["Bonjour", "Monde", "Bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.passed is True
|
||||
assert result.failed_chunk_count == 0
|
||||
assert result.chunk_count == 3
|
||||
|
||||
def test_some_failures(self):
|
||||
result = evaluate_document(
|
||||
["Hello", "World", "Good morning"],
|
||||
["Bonjour", "مرحبا", "Bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.passed is False
|
||||
assert result.failed_chunk_count == 1
|
||||
assert "wrong_script" in result.issues
|
||||
assert len(result.samples) == 1
|
||||
|
||||
def test_empty_lists(self):
|
||||
result = evaluate_document([], [], "fr")
|
||||
assert result.passed is True
|
||||
assert result.chunk_count == 0
|
||||
assert result.score == 0.0
|
||||
|
||||
def test_sample_size_caps_samples(self):
|
||||
# 10 failures, sample_size=3
|
||||
result = evaluate_document(
|
||||
["hi"] * 10,
|
||||
["مرحبا"] * 10,
|
||||
"fr",
|
||||
sample_size=3,
|
||||
)
|
||||
assert result.failed_chunk_count == 10
|
||||
assert len(result.samples) == 3
|
||||
|
||||
def test_score_proportional(self):
|
||||
# 4 chunks, 1 fails → score should be ~0.75 (3/4 passed checks)
|
||||
result = evaluate_document(
|
||||
["hi", "hi", "hi", "hi"],
|
||||
["bonjour", "bonjour", "مرحبا", "bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.failed_chunk_count == 1
|
||||
assert result.score < 1.0
|
||||
assert result.score > 0.0
|
||||
|
||||
|
||||
# ---------- detect_arabic_variant direct ----------
|
||||
|
||||
class TestDetectArabicVariantDirect:
|
||||
def test_empty_text(self):
|
||||
r = detect_arabic_variant("", "fa")
|
||||
assert r["verdict"] == "skip"
|
||||
|
||||
def test_pure_persian_for_persian(self):
|
||||
r = detect_arabic_variant("سلام چطوری", "fa")
|
||||
assert r["verdict"] == "pass"
|
||||
|
||||
def test_pure_persian_for_arabic(self):
|
||||
r = detect_arabic_variant("سلام چطوری", "ar")
|
||||
assert r["verdict"] == "fail"
|
||||
assert "fa" in r["detected_variants"]
|
||||
|
||||
def test_pure_urdu_for_persian(self):
|
||||
r = detect_arabic_variant("السلام ٹڈ", "fa")
|
||||
assert r["verdict"] == "fail"
|
||||
assert "ur" in r["detected_variants"]
|
||||
|
||||
def test_non_arabic_text(self):
|
||||
r = detect_arabic_variant("Bonjour le monde", "fa")
|
||||
# Not in Arabic script → skip
|
||||
assert r["verdict"] == "skip"
|
||||
Reference in New Issue
Block a user