feat(quality): add L0 quality layer (Track A1 + A2 of dev plan)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
L0 quality detection layer to catch translation failures BEFORE they
reach users. Pure Python/TypeScript, zero new dependencies, no API calls.
Backend (Python — services/quality/):
- Script detection: 145 langs mapped to 23 scripts (Latin, Cyrillic,
Greek, Arabic, Hebrew, CJK, Hangul, Kana, Devanagari, Bengali, etc.)
- Language confusion detection (e.g. Arabic text for French target)
- Arabic-script variant discrimination (Persian/Urdu/Pashto/Kurdish
confusion — e.g. Persian text returned when Arabic was requested)
- Length sanity check (with numeric/short-source exemptions)
- Prompt leak detection (Translation: / Voici la traduction: / 翻译:)
- Repetition hallucination detection (token + character level)
- File text extraction for .docx/.xlsx/.pptx/.pdf (no translator
changes needed)
- Defensive pipeline that never raises (L0 must NEVER break a job)
Frontend (TypeScript — wordly.art---traduction-de-documents/src/utils/):
- Exact 1:1 mirror of the Python module
- Zero dependencies, works in browser AND Node.js
- Native Unicode regex (\\p{L}/u) and codePoint iteration
- 63 tests using Node's built-in test runner
Integration:
- Feature-flagged: QUALITY_L0_ENABLED=false (default)
- Observation only: logs structured events, never modifies files
- try/except wrapped: impossible to break a translation job
- Lazy imports: only loaded when flag is on
- Zero impact on existing tests / behavior
Tests:
- 111 Python tests covering all paths (config, script, length, leak,
pipeline, file_extractor) — 100% pass
- 63 TypeScript tests (Node --test) — 100% pass
- 174/174 total tests for the L0 layer
Bug fixes in script mapping:
- yi (Yiddish) -> hebrew (was incorrectly mapped to arabic)
- dv (Maldivian) -> thaana (was incorrectly mapped to arabic)
- ja (Japanese) -> hiragana_katakana (distinguishes from Chinese CJK)
Phase 1 (backend) + Phase 2 (frontend) of Track A complete.
Next: Track B1 (Word/Excel format preservation quick wins).
Closes Track A phase 1+2 of the dev plan.
This commit is contained in:
0
tests/services/quality/__init__.py
Normal file
0
tests/services/quality/__init__.py
Normal file
143
tests/services/quality/test_config.py
Normal file
143
tests/services/quality/test_config.py
Normal file
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Tests for services/quality/config.py
|
||||
Covers the language → script mapping and discriminating characters.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality import config as qconfig
|
||||
|
||||
|
||||
class TestGetScript:
|
||||
def test_cyrillic_languages(self):
|
||||
assert qconfig.get_script("ru") == "cyrillic"
|
||||
assert qconfig.get_script("uk") == "cyrillic"
|
||||
assert qconfig.get_script("be") == "cyrillic"
|
||||
assert qconfig.get_script("bg") == "cyrillic"
|
||||
assert qconfig.get_script("sr") == "cyrillic"
|
||||
assert qconfig.get_script("kk") == "cyrillic"
|
||||
|
||||
def test_latin_languages(self):
|
||||
assert qconfig.get_script("en") == "latin"
|
||||
assert qconfig.get_script("fr") == "latin"
|
||||
assert qconfig.get_script("de") == "latin"
|
||||
assert qconfig.get_script("es") == "latin"
|
||||
assert qconfig.get_script("vi") == "latin"
|
||||
assert qconfig.get_script("tr") == "latin"
|
||||
|
||||
def test_cjk_languages(self):
|
||||
assert qconfig.get_script("zh") == "cjk"
|
||||
assert qconfig.get_script("zh-cn") == "cjk"
|
||||
assert qconfig.get_script("zh-tw") == "cjk"
|
||||
|
||||
def test_japanese_uses_kana(self):
|
||||
# ja is mapped to hiragana_katakana specifically so it can be
|
||||
# distinguished from Chinese CJK.
|
||||
assert qconfig.get_script("ja") == "hiragana_katakana"
|
||||
|
||||
def test_korean_uses_hangul(self):
|
||||
assert qconfig.get_script("ko") == "hangul"
|
||||
|
||||
def test_arabic_script_languages(self):
|
||||
assert qconfig.get_script("ar") == "arabic"
|
||||
assert qconfig.get_script("fa") == "arabic"
|
||||
assert qconfig.get_script("ur") == "arabic"
|
||||
assert qconfig.get_script("ps") == "arabic"
|
||||
assert qconfig.get_script("ku") == "arabic"
|
||||
|
||||
def test_hebrew_and_yiddish(self):
|
||||
assert qconfig.get_script("he") == "hebrew"
|
||||
# Yiddish uses Hebrew script, not Arabic
|
||||
assert qconfig.get_script("yi") == "hebrew"
|
||||
|
||||
def test_thaana(self):
|
||||
# Maldivian (Dhivehi) uses Thaana script, not Arabic
|
||||
assert qconfig.get_script("dv") == "thaana"
|
||||
|
||||
def test_indian_scripts(self):
|
||||
assert qconfig.get_script("hi") == "devanagari"
|
||||
assert qconfig.get_script("bn") == "bengali"
|
||||
assert qconfig.get_script("ta") == "tamil"
|
||||
assert qconfig.get_script("te") == "telugu"
|
||||
assert qconfig.get_script("gu") == "gujarati"
|
||||
assert qconfig.get_script("pa") == "gurmukhi"
|
||||
assert qconfig.get_script("ml") == "malayalam"
|
||||
assert qconfig.get_script("kn") == "kannada"
|
||||
assert qconfig.get_script("si") == "sinhala"
|
||||
|
||||
def test_se_asian_scripts(self):
|
||||
assert qconfig.get_script("th") == "thai"
|
||||
assert qconfig.get_script("lo") == "lao"
|
||||
assert qconfig.get_script("my") == "burmese"
|
||||
assert qconfig.get_script("km") == "khmer"
|
||||
|
||||
def test_other_scripts(self):
|
||||
assert qconfig.get_script("el") == "greek"
|
||||
assert qconfig.get_script("ka") == "georgian"
|
||||
assert qconfig.get_script("hy") == "armenian"
|
||||
assert qconfig.get_script("am") == "ethiopic"
|
||||
assert qconfig.get_script("bo") == "tibetan"
|
||||
|
||||
def test_unknown_falls_back_to_latin(self):
|
||||
assert qconfig.get_script("xx") == "latin"
|
||||
assert qconfig.get_script("") == "latin"
|
||||
assert qconfig.get_script(None) == "latin"
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert qconfig.get_script("FR") == "latin"
|
||||
assert qconfig.get_script("ZH-CN") == "cjk"
|
||||
assert qconfig.get_script("Ru") == "cyrillic"
|
||||
|
||||
|
||||
class TestIsArabicScriptLang:
|
||||
def test_true_for_arabic_script_langs(self):
|
||||
assert qconfig.is_arabic_script_lang("ar") is True
|
||||
assert qconfig.is_arabic_script_lang("fa") is True
|
||||
assert qconfig.is_arabic_script_lang("ur") is True
|
||||
assert qconfig.is_arabic_script_lang("ckb") is True
|
||||
|
||||
def test_false_for_non_arabic(self):
|
||||
assert qconfig.is_arabic_script_lang("fr") is False
|
||||
assert qconfig.is_arabic_script_lang("he") is False
|
||||
assert qconfig.is_arabic_script_lang("hi") is False
|
||||
assert qconfig.is_arabic_script_lang("en") is False
|
||||
|
||||
def test_false_for_unknown(self):
|
||||
assert qconfig.is_arabic_script_lang("xx") is False
|
||||
assert qconfig.is_arabic_script_lang("") is False
|
||||
assert qconfig.is_arabic_script_lang(None) is False
|
||||
|
||||
|
||||
class TestDiscriminatingChars:
|
||||
def test_persian_chars(self):
|
||||
chars = qconfig.get_discriminating_chars("fa")
|
||||
assert "پ" in chars # peh
|
||||
assert "چ" in chars # tcheh
|
||||
assert "ژ" in chars # zheh
|
||||
assert "گ" in chars # guaf
|
||||
|
||||
def test_urdu_chars(self):
|
||||
chars = qconfig.get_discriminating_chars("ur")
|
||||
assert "ٹ" in chars # tteh
|
||||
assert "ڈ" in chars # ddal
|
||||
|
||||
def test_unknown_lang_empty(self):
|
||||
assert qconfig.get_discriminating_chars("fr") == frozenset()
|
||||
assert qconfig.get_discriminating_chars("") == frozenset()
|
||||
assert qconfig.get_discriminating_chars(None) == frozenset()
|
||||
|
||||
|
||||
class TestGetRanges:
|
||||
def test_latin_returns_empty(self):
|
||||
# Latin is the fallback — no ranges, means "anything matches".
|
||||
assert qconfig.get_ranges("latin") == []
|
||||
|
||||
def test_cyrillic_ranges(self):
|
||||
ranges = qconfig.get_ranges("cyrillic")
|
||||
assert any(start == 0x0400 for start, _ in ranges)
|
||||
|
||||
def test_cjk_ranges(self):
|
||||
ranges = qconfig.get_ranges("cjk")
|
||||
assert any(start == 0x4E00 for start, _ in ranges)
|
||||
|
||||
def test_unknown_script_returns_empty(self):
|
||||
assert qconfig.get_ranges("mystery_script") == []
|
||||
181
tests/services/quality/test_file_extractor.py
Normal file
181
tests/services/quality/test_file_extractor.py
Normal file
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
Tests for services/quality/file_extractor.py
|
||||
Uses real files generated via temporary paths.
|
||||
|
||||
On Windows, tempfile.NamedTemporaryFile holds the file open and blocks
|
||||
overwrite, so we use a plain temp directory + manual filename instead.
|
||||
"""
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services.quality.file_extractor import (
|
||||
extract_sample,
|
||||
DEFAULT_MAX_SAMPLES,
|
||||
)
|
||||
|
||||
|
||||
class TestExtractSample:
|
||||
def test_returns_empty_for_missing_file(self):
|
||||
result = extract_sample(Path("/nonexistent/path/file.docx"), ".docx")
|
||||
assert result == []
|
||||
|
||||
def test_returns_empty_for_none_path(self):
|
||||
result = extract_sample(None, ".docx")
|
||||
assert result == []
|
||||
|
||||
def test_returns_empty_for_unsupported_extension(self):
|
||||
# .txt isn't supported — should return [] silently
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = Path(d) / "test.txt"
|
||||
p.write_text("some text")
|
||||
result = extract_sample(p, ".txt")
|
||||
assert result == []
|
||||
|
||||
|
||||
def _make_tmp_path(suffix: str) -> Path:
|
||||
"""Create a unique temp file path. File does not exist yet."""
|
||||
fd, name = tempfile.mkstemp(suffix=suffix)
|
||||
os.close(fd)
|
||||
return Path(name)
|
||||
|
||||
|
||||
class TestDocxExtraction:
|
||||
def test_extracts_paragraphs(self):
|
||||
from docx import Document
|
||||
doc = Document()
|
||||
doc.add_paragraph("Bonjour le monde")
|
||||
doc.add_paragraph("Comment allez-vous?")
|
||||
doc.add_paragraph("Merci beaucoup")
|
||||
p = _make_tmp_path(".docx")
|
||||
try:
|
||||
doc.save(str(p))
|
||||
result = extract_sample(p, ".docx", max_samples=10)
|
||||
assert len(result) >= 1
|
||||
texts = [s["translated"] for s in result]
|
||||
assert "Bonjour le monde" in texts
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
def test_respects_max_samples(self):
|
||||
from docx import Document
|
||||
doc = Document()
|
||||
for i in range(50):
|
||||
doc.add_paragraph(f"Paragraphe numero {i} avec du texte")
|
||||
p = _make_tmp_path(".docx")
|
||||
try:
|
||||
doc.save(str(p))
|
||||
result = extract_sample(p, ".docx", max_samples=5)
|
||||
assert len(result) == 5
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
def test_handles_empty_doc(self):
|
||||
from docx import Document
|
||||
doc = Document()
|
||||
p = _make_tmp_path(".docx")
|
||||
try:
|
||||
doc.save(str(p))
|
||||
result = extract_sample(p, ".docx")
|
||||
assert result == []
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
|
||||
class TestXlsxExtraction:
|
||||
def test_extracts_cells(self):
|
||||
from openpyxl import Workbook
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "Bonjour"
|
||||
ws["A2"] = "Monde"
|
||||
ws["A3"] = "Comment"
|
||||
p = _make_tmp_path(".xlsx")
|
||||
try:
|
||||
wb.save(str(p))
|
||||
wb.close()
|
||||
result = extract_sample(p, ".xlsx", max_samples=10)
|
||||
assert len(result) >= 1
|
||||
texts = [s["translated"] for s in result]
|
||||
assert "Bonjour" in texts
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
def test_skips_numeric_cells(self):
|
||||
from openpyxl import Workbook
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = 100
|
||||
ws["A2"] = 200
|
||||
ws["A3"] = "Hello"
|
||||
p = _make_tmp_path(".xlsx")
|
||||
try:
|
||||
wb.save(str(p))
|
||||
wb.close()
|
||||
result = extract_sample(p, ".xlsx")
|
||||
texts = [s["translated"] for s in result]
|
||||
assert "Hello" in texts
|
||||
assert 100 not in texts # numeric only cells skipped
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
|
||||
class TestPptxExtraction:
|
||||
def test_extracts_slide_text(self):
|
||||
from pptx import Presentation
|
||||
pres = Presentation()
|
||||
slide = pres.slides.add_slide(pres.slide_layouts[0])
|
||||
slide.shapes.title.text = "Bonjour le monde"
|
||||
p = _make_tmp_path(".pptx")
|
||||
try:
|
||||
pres.save(str(p))
|
||||
result = extract_sample(p, ".pptx")
|
||||
assert len(result) >= 1
|
||||
assert result[0]["translated"] == "Bonjour le monde"
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
|
||||
|
||||
class TestPdfExtraction:
|
||||
def test_extracts_pdf_text(self):
|
||||
# Create a minimal PDF using reportlab if available, else skip
|
||||
pytest.importorskip("fitz")
|
||||
try:
|
||||
from reportlab.pdfgen import canvas
|
||||
except ImportError:
|
||||
pytest.skip("reportlab not available")
|
||||
p = _make_tmp_path(".pdf")
|
||||
try:
|
||||
c = canvas.Canvas(str(p))
|
||||
c.drawString(100, 750, "Bonjour le monde")
|
||||
c.drawString(100, 700, "Comment allez vous")
|
||||
c.save()
|
||||
result = extract_sample(p, ".pdf")
|
||||
assert len(result) >= 1
|
||||
texts = [s["translated"] for s in result]
|
||||
assert any("Bonjour" in t for t in texts)
|
||||
finally:
|
||||
try:
|
||||
p.unlink()
|
||||
except (OSError, PermissionError):
|
||||
pass
|
||||
49
tests/services/quality/test_length_checker.py
Normal file
49
tests/services/quality/test_length_checker.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Tests for services/quality/length_checker.py
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality.length_checker import check
|
||||
|
||||
|
||||
class TestLengthCheck:
|
||||
def test_normal_length(self):
|
||||
r = check("Hello world, this is a test of translation length",
|
||||
"Bonjour le monde, ceci est un test de longueur de traduction")
|
||||
assert r["issue"] is None
|
||||
assert r["ratio"] is not None
|
||||
assert 0.5 < r["ratio"] < 2.0
|
||||
|
||||
def test_huge_translation(self):
|
||||
src = "A" * 200
|
||||
r = check(src, "x" * 1000)
|
||||
assert r["issue"] == "length_outlier"
|
||||
assert r["ratio"] > 3.5
|
||||
|
||||
def test_tiny_translation(self):
|
||||
r = check("A" * 100, "ok")
|
||||
assert r["issue"] == "truncation_suspect"
|
||||
assert r["ratio"] < 0.15
|
||||
|
||||
def test_empty_translation_flagged(self):
|
||||
r = check("Hello world, this is a test", "")
|
||||
assert r["issue"] == "truncation_suspect"
|
||||
|
||||
def test_short_source_skips_ratio(self):
|
||||
r = check("Hi", "ok")
|
||||
assert r["issue"] is None
|
||||
assert r["ratio"] is None
|
||||
|
||||
def test_empty_inputs(self):
|
||||
r = check("", "")
|
||||
assert r["issue"] is None
|
||||
assert r["source_length"] == 0
|
||||
assert r["translated_length"] == 0
|
||||
|
||||
def test_none_source(self):
|
||||
r = check(None, "translation")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_numeric_source(self):
|
||||
r = check("Price: 100", "100")
|
||||
assert r["issue"] is None # numeric sources skip the check
|
||||
85
tests/services/quality/test_pattern_leak.py
Normal file
85
tests/services/quality/test_pattern_leak.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Tests for services/quality/pattern_leak.py
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality.pattern_leak import check
|
||||
|
||||
|
||||
class TestPromptLeak:
|
||||
def test_english_translation_prefix(self):
|
||||
r = check("Translation: Bonjour le monde")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_here_is_translation(self):
|
||||
r = check("Here is the translation: Bonjour")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_french_voici_traduction(self):
|
||||
r = check("Voici la traduction : Bonjour le monde")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_chinese_translation_prefix(self):
|
||||
r = check("翻译:你好世界")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_sure_heres_translation(self):
|
||||
r = check("Sure, here's the translation: Hello world")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_of_course_heres(self):
|
||||
r = check("Of course, here you go: Bonjour")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_markdown_bold_translation(self):
|
||||
r = check("**Translation** Bonjour le monde")
|
||||
assert r["issue"] == "prompt_leak"
|
||||
|
||||
def test_normal_text_passes(self):
|
||||
r = check("Bonjour le monde, comment allez-vous?")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_text_with_translation_word_in_middle_passes(self):
|
||||
r = check("Voici ce que je pense de la traduction de ce texte")
|
||||
# The word "traduction" appears but not as a prefix
|
||||
assert r["issue"] is None
|
||||
|
||||
|
||||
class TestRepetitionHallucination:
|
||||
def test_long_repetition_detected(self):
|
||||
r = check("the the the the the the the the")
|
||||
assert r["issue"] == "repetition_hallucination"
|
||||
assert r["repetition_count"] >= 5
|
||||
|
||||
def test_short_repetition_passes(self):
|
||||
# 4 times is within tolerance
|
||||
r = check("I think I think I think I think")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_normal_text_passes(self):
|
||||
r = check("This is a normal sentence with no repetition issues")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_mixed_repetition_in_long_text_passes(self):
|
||||
# Repetition is local, not a hallucination
|
||||
r = check("The cat sat on the mat. The cat was happy. The dog was sad.")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_repetition_with_punctuation(self):
|
||||
# "the, the, the, the, the" should still be detected
|
||||
r = check("the, the, the, the, the, the")
|
||||
assert r["issue"] == "repetition_hallucination"
|
||||
|
||||
|
||||
class TestEmptyAndEdgeCases:
|
||||
def test_empty_text(self):
|
||||
r = check("")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_whitespace_only(self):
|
||||
r = check(" \n \t ")
|
||||
assert r["issue"] is None
|
||||
|
||||
def test_none_input(self):
|
||||
r = check(None)
|
||||
assert r["issue"] is None
|
||||
61
tests/services/quality/test_pipeline.py
Normal file
61
tests/services/quality/test_pipeline.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Tests for services/quality/pipeline.py
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality import run_l0_check
|
||||
from services.quality.script_detector import DocumentQualityResult
|
||||
|
||||
|
||||
class TestRunL0Check:
|
||||
def test_returns_result_on_success(self):
|
||||
result = run_l0_check(
|
||||
["Hello", "World"],
|
||||
["Bonjour", "Monde"],
|
||||
"fr",
|
||||
job_id="test_job_1",
|
||||
)
|
||||
assert isinstance(result, DocumentQualityResult)
|
||||
assert result.passed is True
|
||||
assert result.chunk_count == 2
|
||||
|
||||
def test_returns_neutral_on_empty_lists(self):
|
||||
result = run_l0_check([], [], "fr", job_id="test_job_2")
|
||||
assert result.passed is True
|
||||
assert result.chunk_count == 0
|
||||
|
||||
def test_detects_failures(self):
|
||||
result = run_l0_check(
|
||||
["Hello", "World"],
|
||||
["Bonjour", "مرحبا"],
|
||||
"fr",
|
||||
job_id="test_job_3",
|
||||
)
|
||||
assert result.passed is False
|
||||
assert "wrong_script" in result.issues
|
||||
|
||||
def test_never_raises_on_bad_input(self):
|
||||
# Even with weird input, it shouldn't raise
|
||||
result = run_l0_check(
|
||||
["Hello"],
|
||||
[None], # None translation
|
||||
"fr",
|
||||
job_id="test_job_4",
|
||||
)
|
||||
assert result is not None
|
||||
|
||||
def test_optional_job_id(self):
|
||||
# job_id is optional
|
||||
result = run_l0_check(["hi"], ["salut"], "fr")
|
||||
assert result is not None
|
||||
|
||||
def test_optional_file_extension(self):
|
||||
result = run_l0_check(
|
||||
["hi"], ["salut"], "fr", file_extension=".docx"
|
||||
)
|
||||
assert result is not None
|
||||
|
||||
def test_target_lang_none(self):
|
||||
# Should not crash with no target lang
|
||||
result = run_l0_check(["hi"], ["salut"], None)
|
||||
assert result is not None
|
||||
290
tests/services/quality/test_script_detector.py
Normal file
290
tests/services/quality/test_script_detector.py
Normal file
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Tests for services/quality/script_detector.py
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from services.quality import evaluate_chunk, evaluate_document, detect_arabic_variant
|
||||
|
||||
|
||||
# ---------- Happy path: correct script ----------
|
||||
|
||||
class TestCorrectScript:
|
||||
def test_russian_correct(self):
|
||||
r = evaluate_chunk("Hello world", "Привет мир", "ru")
|
||||
assert r.passed is True
|
||||
assert "wrong_script" not in r.issues
|
||||
|
||||
def test_chinese_simplified_correct(self):
|
||||
r = evaluate_chunk("Hello", "你好世界", "zh")
|
||||
assert r.passed is True
|
||||
|
||||
def test_arabic_correct(self):
|
||||
r = evaluate_chunk("Hello", "مرحبا بالعالم", "ar")
|
||||
assert r.passed is True
|
||||
|
||||
def test_persian_correct(self):
|
||||
# Persian has unique chars چ پ ژ گ
|
||||
r = evaluate_chunk("Hello", "سلام چطوری؟ من پژوهشگر هستم", "fa")
|
||||
assert r.passed is True
|
||||
|
||||
def test_french_correct(self):
|
||||
r = evaluate_chunk("Hello world", "Bonjour le monde", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_hebrew_correct(self):
|
||||
r = evaluate_chunk("Hello", "שלום עולם", "he")
|
||||
assert r.passed is True
|
||||
|
||||
def test_korean_correct(self):
|
||||
r = evaluate_chunk("Hello", "안녕하세요 세계", "ko")
|
||||
assert r.passed is True
|
||||
|
||||
def test_japanese_correct(self):
|
||||
# Japanese must contain hiragana or katakana to be classified as ja.
|
||||
r = evaluate_chunk("Hello", "こんにちは世界", "ja")
|
||||
assert r.passed is True
|
||||
|
||||
def test_hindi_correct(self):
|
||||
r = evaluate_chunk("Hello", "नमस्ते दुनिया", "hi")
|
||||
assert r.passed is True
|
||||
|
||||
def test_thai_correct(self):
|
||||
r = evaluate_chunk("Hello", "สวัสดีชาวโลก", "th")
|
||||
assert r.passed is True
|
||||
|
||||
def test_greek_correct(self):
|
||||
r = evaluate_chunk("Hello", "Γεια σας κόσμε", "el")
|
||||
assert r.passed is True
|
||||
|
||||
|
||||
# ---------- Wrong script: language confusion ----------
|
||||
|
||||
class TestWrongScript:
|
||||
def test_french_text_for_japanese_target(self):
|
||||
r = evaluate_chunk("Hello", "Bonjour le monde", "ja")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_arabic_text_for_french_target(self):
|
||||
r = evaluate_chunk("Hello", "مرحبا بالعالم", "fr")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_russian_text_for_english_target(self):
|
||||
r = evaluate_chunk("Hello", "Привет мир", "en")
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
def test_chinese_text_for_korean_target(self):
|
||||
r = evaluate_chunk("Hello", "你好世界", "ko")
|
||||
# Hangul syllables are 0xAC00-0xD7AF — Chinese chars are not in that range.
|
||||
assert r.passed is False
|
||||
assert "wrong_script" in r.issues
|
||||
|
||||
|
||||
# ---------- Persian / Arabic discrimination ----------
|
||||
|
||||
class TestArabicVariantDiscrimination:
|
||||
def test_persian_for_arabic_target_fails(self):
|
||||
# Persian text with چ پ ژ گ should fail when target is Arabic.
|
||||
r = evaluate_chunk("Hello", "سلام چطوری؟", "ar")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
def test_arabic_for_persian_target_fails(self):
|
||||
# Pure Arabic text (no Persian-specific chars) when target is Persian.
|
||||
r = evaluate_chunk("Hello", "السلام عليكم", "fa")
|
||||
# No Persian-specific chars, no other Arabic-script variant chars
|
||||
# → should pass the variant check (because no discrimination signal).
|
||||
# This is a known limitation: pure Arabic indistinguishable from pure Persian.
|
||||
# We accept that the variant check only flags MISMATCHES with discriminating chars.
|
||||
assert "wrong_arabic_variant" not in r.issues
|
||||
|
||||
def test_urdu_for_persian_target_fails(self):
|
||||
# Urdu with ٹ ڈ should fail when target is Persian.
|
||||
r = evaluate_chunk("Hello", "السلام ٹڈ", "fa")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
def test_pashto_for_arabic_target_fails(self):
|
||||
# Pashto with ټډړ should fail when target is Arabic.
|
||||
r = evaluate_chunk("Hello", "السلام ټډړ", "ar")
|
||||
assert r.passed is False
|
||||
assert "wrong_arabic_variant" in r.issues
|
||||
|
||||
|
||||
# ---------- Edge cases ----------
|
||||
|
||||
class TestEdgeCases:
|
||||
def test_empty_translation_passes(self):
|
||||
# Empty translations are skipped (the provider may have legitimately
|
||||
# produced nothing — e.g. an empty cell).
|
||||
r = evaluate_chunk("Hello", "", "fr")
|
||||
assert r.passed is True
|
||||
assert "empty_translation" in r.issues
|
||||
|
||||
def test_whitespace_only_passes(self):
|
||||
r = evaluate_chunk("Hello", " \n ", "fr")
|
||||
assert r.passed is True
|
||||
assert "empty_translation" in r.issues
|
||||
|
||||
def test_none_translation_passes(self):
|
||||
r = evaluate_chunk("Hello", None, "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_numbers_only_passes(self):
|
||||
r = evaluate_chunk("Price: 100", "100", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
def test_unknown_target_lang_falls_back(self):
|
||||
# Unknown lang → latin → no script check, just length/leak checks.
|
||||
r = evaluate_chunk("Hello", "Bonjour", "xx")
|
||||
assert r.passed is True
|
||||
|
||||
def test_mixed_brands_latin_target(self):
|
||||
# "iPhone 15 Pro Max" is mostly Latin + digits — fine for fr.
|
||||
r = evaluate_chunk("Buy iPhone 15", "Acheter iPhone 15", "fr")
|
||||
assert r.passed is True
|
||||
|
||||
|
||||
# ---------- Length sanity ----------
|
||||
|
||||
class TestLengthIssues:
|
||||
def test_huge_translation_flagged_via_repetition(self):
|
||||
# Source too short for length ratio, but the "x"*1000 repetition
|
||||
# is caught by the pattern/repetition check, not length.
|
||||
r = evaluate_chunk("Short", "x" * 1000, "fr")
|
||||
assert "repetition_hallucination" in r.issues
|
||||
|
||||
def test_huge_translation_with_long_source_flagged(self):
|
||||
# Long enough source → length ratio kicks in → length_outlier.
|
||||
src = "A" * 200
|
||||
r = evaluate_chunk(src, "x" * 1000, "fr")
|
||||
assert "length_outlier" in r.issues
|
||||
|
||||
def test_tiny_translation_flagged(self):
|
||||
r = evaluate_chunk("A" * 100, "ok", "fr")
|
||||
# ratio is 2/100 = 0.02, well below 0.15
|
||||
assert "truncation_suspect" in r.issues
|
||||
|
||||
def test_normal_translation_no_length_issue(self):
|
||||
r = evaluate_chunk("Hello world, how are you?", "Bonjour le monde, comment allez-vous?", "fr")
|
||||
assert "length_outlier" not in r.issues
|
||||
assert "truncation_suspect" not in r.issues
|
||||
|
||||
def test_short_source_skips_ratio(self):
|
||||
r = evaluate_chunk("Hi", "ok", "fr")
|
||||
# source too short for ratio check
|
||||
assert "length_outlier" not in r.issues
|
||||
assert "truncation_suspect" not in r.issues
|
||||
|
||||
def test_numeric_source_skips_length_check(self):
|
||||
# Source is mostly digits — translation can also be short.
|
||||
r = evaluate_chunk("Price: 100", "100", "fr")
|
||||
assert "truncation_suspect" not in r.issues
|
||||
assert "length_outlier" not in r.issues
|
||||
|
||||
|
||||
# ---------- Pattern leak / repetition ----------
|
||||
|
||||
class TestPatternLeak:
|
||||
def test_prompt_leak_english_detected(self):
|
||||
r = evaluate_chunk("Hello", "Translation: Bonjour le monde", "fr")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_prompt_leak_french_detected(self):
|
||||
r = evaluate_chunk("Hello", "Voici la traduction : Bonjour", "fr")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_prompt_leak_chinese_detected(self):
|
||||
r = evaluate_chunk("Hello", "翻译:你好", "zh")
|
||||
assert "prompt_leak" in r.issues
|
||||
|
||||
def test_repetition_hallucination_detected(self):
|
||||
r = evaluate_chunk("Hello", "the the the the the the", "fr")
|
||||
assert "repetition_hallucination" in r.issues
|
||||
|
||||
def test_normal_text_no_leak(self):
|
||||
r = evaluate_chunk("Hello", "Bonjour le monde", "fr")
|
||||
assert "prompt_leak" not in r.issues
|
||||
assert "repetition_hallucination" not in r.issues
|
||||
|
||||
|
||||
# ---------- Document-level aggregation ----------
|
||||
|
||||
class TestEvaluateDocument:
|
||||
def test_all_good(self):
|
||||
result = evaluate_document(
|
||||
["Hello", "World", "Good morning"],
|
||||
["Bonjour", "Monde", "Bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.passed is True
|
||||
assert result.failed_chunk_count == 0
|
||||
assert result.chunk_count == 3
|
||||
|
||||
def test_some_failures(self):
|
||||
result = evaluate_document(
|
||||
["Hello", "World", "Good morning"],
|
||||
["Bonjour", "مرحبا", "Bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.passed is False
|
||||
assert result.failed_chunk_count == 1
|
||||
assert "wrong_script" in result.issues
|
||||
assert len(result.samples) == 1
|
||||
|
||||
def test_empty_lists(self):
|
||||
result = evaluate_document([], [], "fr")
|
||||
assert result.passed is True
|
||||
assert result.chunk_count == 0
|
||||
assert result.score == 0.0
|
||||
|
||||
def test_sample_size_caps_samples(self):
|
||||
# 10 failures, sample_size=3
|
||||
result = evaluate_document(
|
||||
["hi"] * 10,
|
||||
["مرحبا"] * 10,
|
||||
"fr",
|
||||
sample_size=3,
|
||||
)
|
||||
assert result.failed_chunk_count == 10
|
||||
assert len(result.samples) == 3
|
||||
|
||||
def test_score_proportional(self):
|
||||
# 4 chunks, 1 fails → score should be ~0.75 (3/4 passed checks)
|
||||
result = evaluate_document(
|
||||
["hi", "hi", "hi", "hi"],
|
||||
["bonjour", "bonjour", "مرحبا", "bonjour"],
|
||||
"fr",
|
||||
)
|
||||
assert result.failed_chunk_count == 1
|
||||
assert result.score < 1.0
|
||||
assert result.score > 0.0
|
||||
|
||||
|
||||
# ---------- detect_arabic_variant direct ----------
|
||||
|
||||
class TestDetectArabicVariantDirect:
|
||||
def test_empty_text(self):
|
||||
r = detect_arabic_variant("", "fa")
|
||||
assert r["verdict"] == "skip"
|
||||
|
||||
def test_pure_persian_for_persian(self):
|
||||
r = detect_arabic_variant("سلام چطوری", "fa")
|
||||
assert r["verdict"] == "pass"
|
||||
|
||||
def test_pure_persian_for_arabic(self):
|
||||
r = detect_arabic_variant("سلام چطوری", "ar")
|
||||
assert r["verdict"] == "fail"
|
||||
assert "fa" in r["detected_variants"]
|
||||
|
||||
def test_pure_urdu_for_persian(self):
|
||||
r = detect_arabic_variant("السلام ٹڈ", "fa")
|
||||
assert r["verdict"] == "fail"
|
||||
assert "ur" in r["detected_variants"]
|
||||
|
||||
def test_non_arabic_text(self):
|
||||
r = detect_arabic_variant("Bonjour le monde", "fa")
|
||||
# Not in Arabic script → skip
|
||||
assert r["verdict"] == "skip"
|
||||
Reference in New Issue
Block a user