sepehr
4d466699fd
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m26s
L1 quality layer — uses a cheap LLM via the OpenAI-compatible API to
validate translation quality. Designed to be the SECOND line of defense
after L0 (script detection, length, pattern).
Architecture:
- sampler.py — picks 5 representative chunks per job (longest first,
skips L0-failed indices, skips too-short or identical pairs)
- llm_judge.py — OpenAI-compatible client, binary verdict per chunk
(accurate / fluent / correct_language / no_leaks), JSON output,
hard timeout, defensive (never raises), cost estimation built in
- pipeline.py — defensive wrapper that integrates both, never breaks
a translation job, always logs a structured event
Integration:
- 5 feature flags in config.py (QUALITY_L1_ENABLED, _LOG_ONLY, etc.)
- QUALITY_L1_LOG_ONLY=true by default: log-only mode, verdict NEVER
blocks or retries a job
- Reuses the chunks extracted by L0 (no double work)
- Passes the set of L0-failed indices so L1 doesn't re-judge them
- Wrapped in try/except so a misconfigured L1 NEVER breaks a job
Default config: deepseek-chat via DeepSeek API
- Cost: ~0.0003 USD per job (5 chunks)
- Speed: typically 1-2s per call, hard ceiling at 8s
- Easy to swap: just set L1_JUDGE_BASE_URL and L1_JUDGE_MODEL
LLM judge is intentionally a SEPARATE model from the translator
(self-evaluation bias mitigation — Meta/Stanford papers 2024-2025).
Tests:
test_sampler.py — 9 tests covering the sampling strategy
test_llm_judge.py — 22 tests covering init, parsing, mocked API,
cost estimation, env factory
test_l1_pipeline.py — 6 tests covering the wrapper
Total new: 37 tests, all pass
Grand total quality+format: 264 tests passing (0 regression)
All 36 new tests + 111 L0 tests + 117 existing translator tests = 264
Phase 1 (observation) for 2 weeks. Then QUALITY_L1_LOG_ONLY=false
to enable auto-retry via the fallback chain.
102 lines
4.9 KiB
Python
102 lines
4.9 KiB
Python
"""
|
|
Tests for services/quality/sampler.py
|
|
"""
|
|
import pytest
|
|
|
|
from services.quality.sampler import sample_chunks_for_l1
|
|
|
|
|
|
class TestSampleChunksForL1:
|
|
def test_empty_chunks(self):
|
|
result = sample_chunks_for_l1([], [], set(), max_samples=5, min_chunks=10)
|
|
assert result == []
|
|
|
|
def test_too_few_chunks(self):
|
|
sources = ["Hello", "World", "Goodbye"]
|
|
translations = ["Bonjour", "Monde", "Au revoir"]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=5, min_chunks=10)
|
|
# Only 3 chunks, min_chunks=10 → no sample
|
|
assert result == []
|
|
|
|
def test_min_chunks_exact(self):
|
|
sources = ["chunk"] * 10
|
|
translations = ["morceau"] * 10
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=5, min_chunks=10)
|
|
# Exactly 10 chunks, min_chunks=10 → should sample
|
|
assert len(result) == 5
|
|
|
|
def test_skips_l0_failures(self):
|
|
sources = ["a" * 100, "b" * 200, "c" * 300, "d" * 400]
|
|
translations = ["x" * 100, "y" * 200, "z" * 300, "w" * 400]
|
|
# Mark index 0 and 2 as L0 failures
|
|
result = sample_chunks_for_l1(sources, translations, {0, 2},
|
|
max_samples=10, min_chunks=3)
|
|
sources_returned = [s for s, t in result]
|
|
assert sources_returned == ["b" * 200, "d" * 400]
|
|
|
|
def test_prefers_longest_chunks(self):
|
|
# 10 chunks of different lengths. Each chunk has a unique marker
|
|
# so we can verify which were picked regardless of whitespace.
|
|
sources = [f"src{i}_" + "x" * (i * 5) for i in range(10)]
|
|
translations = [f"tr{i}_" + "y" * (i * 5) for i in range(10)]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=3, min_chunks=5)
|
|
# The 3 longest should be picked (indices 9, 8, 7)
|
|
assert len(result) == 3
|
|
# Verify the longest chunks were picked
|
|
markers_returned = [s for s, t in result]
|
|
assert any("src9_" in s for s in markers_returned)
|
|
assert any("src8_" in s for s in markers_returned)
|
|
assert any("src7_" in s for s in markers_returned)
|
|
# And the shortest were NOT picked
|
|
assert not any("src0_" in s for s in markers_returned)
|
|
assert not any("src1_" in s for s in markers_returned)
|
|
|
|
def test_skips_identical_source_translation(self):
|
|
# Identical pairs are probably numbers, codes, brand names
|
|
sources = ["12345", "Hello", "WORLD"]
|
|
translations = ["12345", "Bonjour", "WORLD"]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=5, min_chunks=3)
|
|
# "12345" and "WORLD" pairs should be skipped (identical)
|
|
sources_returned = [s for s, t in result]
|
|
assert "12345" not in sources_returned
|
|
assert "WORLD" not in sources_returned
|
|
assert "Hello" in sources_returned
|
|
|
|
def test_skips_very_short_pairs(self):
|
|
# Very short pairs don't have enough context
|
|
sources = ["a", "Hello world this is a longer test sentence",
|
|
"b", "Another longer sentence for testing purposes"]
|
|
translations = ["x", "Bonjour le monde ceci est une phrase plus longue",
|
|
"y", "Une autre phrase plus longue pour tester"]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=5, min_chunks=2)
|
|
# The "a"/"x" and "b"/"y" pairs should be skipped
|
|
sources_returned = [s for s, t in result]
|
|
assert "a" not in sources_returned
|
|
assert "b" not in sources_returned
|
|
assert len(result) == 2
|
|
|
|
def test_respects_max_samples(self):
|
|
sources = [f"long source {i} " * 10 for i in range(20)]
|
|
translations = [f"long translation {i} " * 10 for i in range(20)]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=3, min_chunks=5)
|
|
assert len(result) == 3
|
|
|
|
def test_results_in_document_order(self):
|
|
# The function should return in original document order,
|
|
# not in the length-priority order. (Use .strip()-friendly data
|
|
# to avoid whitespace edge cases in equality.)
|
|
sources = ["short source 1", "long source 2 " * 20, "medium source 3 " * 10, "tiny source 4"]
|
|
translations = ["court 1", "longue 2 " * 20, "moyen 3 " * 10, "minuscule 4"]
|
|
result = sample_chunks_for_l1(sources, translations, set(),
|
|
max_samples=4, min_chunks=2)
|
|
# Indices in the result should be 0, 1, 2, 3 (in document order)
|
|
for i, (s, t) in enumerate(result):
|
|
assert s == sources[i].strip(), f"Source {i} mismatch: {s!r} vs {sources[i].strip()!r}"
|
|
assert t == translations[i].strip(), f"Translation {i} mismatch"
|