sepehr
4d466699fd
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m26s
L1 quality layer — uses a cheap LLM via the OpenAI-compatible API to
validate translation quality. Designed to be the SECOND line of defense
after L0 (script detection, length, pattern).
Architecture:
- sampler.py — picks 5 representative chunks per job (longest first,
skips L0-failed indices, skips too-short or identical pairs)
- llm_judge.py — OpenAI-compatible client, binary verdict per chunk
(accurate / fluent / correct_language / no_leaks), JSON output,
hard timeout, defensive (never raises), cost estimation built in
- pipeline.py — defensive wrapper that integrates both, never breaks
a translation job, always logs a structured event
Integration:
- 5 feature flags in config.py (QUALITY_L1_ENABLED, _LOG_ONLY, etc.)
- QUALITY_L1_LOG_ONLY=true by default: log-only mode, verdict NEVER
blocks or retries a job
- Reuses the chunks extracted by L0 (no double work)
- Passes the set of L0-failed indices so L1 doesn't re-judge them
- Wrapped in try/except so a misconfigured L1 NEVER breaks a job
Default config: deepseek-chat via DeepSeek API
- Cost: ~0.0003 USD per job (5 chunks)
- Speed: typically 1-2s per call, hard ceiling at 8s
- Easy to swap: just set L1_JUDGE_BASE_URL and L1_JUDGE_MODEL
LLM judge is intentionally a SEPARATE model from the translator
(self-evaluation bias mitigation — Meta/Stanford papers 2024-2025).
Tests:
test_sampler.py — 9 tests covering the sampling strategy
test_llm_judge.py — 22 tests covering init, parsing, mocked API,
cost estimation, env factory
test_l1_pipeline.py — 6 tests covering the wrapper
Total new: 37 tests, all pass
Grand total quality+format: 264 tests passing (0 regression)
All 36 new tests + 111 L0 tests + 117 existing translator tests = 264
Phase 1 (observation) for 2 weeks. Then QUALITY_L1_LOG_ONLY=false
to enable auto-retry via the fallback chain.
259 lines
9.6 KiB
Python
259 lines
9.6 KiB
Python
"""
|
|
Tests for services/quality/llm_judge.py
|
|
Uses mocks for the OpenAI client — no actual API calls.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from services.quality.llm_judge import (
|
|
LLMJudge,
|
|
L1Result,
|
|
L1ChunkVerdict,
|
|
JUDGE_SYSTEM_PROMPT,
|
|
make_judge_from_env,
|
|
)
|
|
|
|
|
|
# ---------- Judge init ----------
|
|
|
|
class TestLLMJudgeInit:
|
|
def test_requires_api_key(self):
|
|
with pytest.raises(ValueError):
|
|
LLMJudge(api_key="")
|
|
|
|
def test_init_with_defaults(self):
|
|
judge = LLMJudge(api_key="test-key")
|
|
assert judge._api_key == "test-key"
|
|
assert judge._model == "deepseek-chat"
|
|
assert judge._base_url == "https://api.deepseek.com/v1"
|
|
|
|
def test_init_with_custom_params(self):
|
|
judge = LLMJudge(
|
|
api_key="k",
|
|
base_url="https://api.openai.com/v1/",
|
|
model="gpt-4o-mini",
|
|
timeout_seconds=15.0,
|
|
)
|
|
assert judge._base_url == "https://api.openai.com/v1" # trailing slash stripped
|
|
assert judge._model == "gpt-4o-mini"
|
|
assert judge._timeout == 15.0
|
|
|
|
|
|
# ---------- Response parsing ----------
|
|
|
|
class TestParseResponse:
|
|
def setup_method(self):
|
|
self.judge = LLMJudge(api_key="test-key")
|
|
|
|
def _make_response(self, content: str):
|
|
response = MagicMock()
|
|
response.choices = [MagicMock()]
|
|
response.choices[0].message.content = content
|
|
return response
|
|
|
|
def test_parse_pure_json_array(self):
|
|
content = json.dumps([
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "good"}
|
|
])
|
|
verdicts = self.judge._parse_response(self._make_response(content), 1)
|
|
assert len(verdicts) == 1
|
|
assert verdicts[0].passed is True
|
|
assert verdicts[0].accurate is True
|
|
assert verdicts[0].reason == "good"
|
|
|
|
def test_parse_json_object_with_verdicts_key(self):
|
|
content = json.dumps({"verdicts": [
|
|
{"accurate": "no", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "lost meaning"}
|
|
]})
|
|
verdicts = self.judge._parse_response(self._make_response(content), 1)
|
|
assert len(verdicts) == 1
|
|
assert verdicts[0].passed is False
|
|
assert verdicts[0].accurate is False
|
|
assert verdicts[0].fluent is True
|
|
assert verdicts[0].reason == "lost meaning"
|
|
|
|
def test_parse_json_with_markdown_fences(self):
|
|
content = "```json\n" + json.dumps([
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "ok"}
|
|
]) + "\n```"
|
|
verdicts = self.judge._parse_response(self._make_response(content), 1)
|
|
assert len(verdicts) == 1
|
|
assert verdicts[0].passed is True
|
|
|
|
def test_parse_invalid_json_returns_empty(self):
|
|
content = "not json at all"
|
|
verdicts = self.judge._parse_response(self._make_response(content), 1)
|
|
assert verdicts == []
|
|
|
|
def test_parse_partial_verdict_defaults_to_false(self):
|
|
content = json.dumps([{"accurate": "yes"}]) # missing other fields
|
|
verdicts = self.judge._parse_response(self._make_response(content), 1)
|
|
assert len(verdicts) == 1
|
|
# All fields default to False when missing
|
|
assert verdicts[0].passed is False
|
|
assert verdicts[0].accurate is True
|
|
assert verdicts[0].fluent is False
|
|
assert verdicts[0].correct_language is False
|
|
assert verdicts[0].no_leaks is False
|
|
|
|
def test_parse_mixed_pass_fail(self):
|
|
content = json.dumps([
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "good"},
|
|
{"accurate": "no", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "mistranslation"},
|
|
{"accurate": "yes", "fluent": "no", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "awkward"},
|
|
])
|
|
verdicts = self.judge._parse_response(self._make_response(content), 3)
|
|
assert len(verdicts) == 3
|
|
assert verdicts[0].passed is True
|
|
assert verdicts[1].passed is False
|
|
assert verdicts[2].passed is False
|
|
|
|
|
|
# ---------- L1Result ----------
|
|
|
|
class TestL1Result:
|
|
def test_passed_property(self):
|
|
v = L1ChunkVerdict(accurate=True, fluent=True, correct_language=True, no_leaks=True)
|
|
assert v.passed is True
|
|
|
|
v_fail = L1ChunkVerdict(accurate=True, fluent=True, correct_language=True, no_leaks=False)
|
|
assert v_fail.passed is False
|
|
|
|
|
|
# ---------- Cost estimation ----------
|
|
|
|
class TestCostEstimation:
|
|
def test_deepseek_estimate(self):
|
|
judge = LLMJudge(api_key="k", model="deepseek-chat")
|
|
cost = judge._estimate_cost(5)
|
|
# 5 pairs: ~1450 input + 250 output tokens
|
|
# Cost should be tiny (< $0.01)
|
|
assert 0.0 < cost < 0.01
|
|
|
|
def test_gpt4o_mini_more_expensive(self):
|
|
judge_ds = LLMJudge(api_key="k", model="deepseek-chat")
|
|
judge_gpt = LLMJudge(api_key="k", model="gpt-4o-mini")
|
|
assert judge_gpt._estimate_cost(5) > judge_ds._estimate_cost(5)
|
|
|
|
def test_gemini_flash_cheaper(self):
|
|
judge_gemini = LLMJudge(api_key="k", model="gemini-2.5-flash-lite")
|
|
judge_gpt = LLMJudge(api_key="k", model="gpt-4o-mini")
|
|
assert judge_gemini._estimate_cost(5) < judge_gpt._estimate_cost(5)
|
|
|
|
|
|
# ---------- Judge batch (mocked API) ----------
|
|
|
|
class TestJudgeBatch:
|
|
"""Tests the full judge_batch flow with a mocked OpenAI client."""
|
|
|
|
def setup_method(self):
|
|
self.judge = LLMJudge(api_key="test-key", timeout_seconds=5.0)
|
|
|
|
def _mock_response_with_verdicts(self, verdicts_data):
|
|
response = MagicMock()
|
|
response.choices = [MagicMock()]
|
|
response.choices[0].message.content = json.dumps(verdicts_data)
|
|
return response
|
|
|
|
def _run_judge_batch(self, verdicts_data):
|
|
"""Helper to run judge_batch with a mocked client."""
|
|
# Build the mock client BEFORE calling the method
|
|
response = self._mock_response_with_verdicts(verdicts_data)
|
|
mock_client = MagicMock()
|
|
mock_client.chat = MagicMock()
|
|
mock_client.chat.completions = MagicMock()
|
|
mock_client.chat.completions.create = AsyncMock(return_value=response)
|
|
self.judge._client = mock_client
|
|
return asyncio.run(self.judge.judge_batch(
|
|
[("Source 1", "Translation 1"), ("Source 2", "Translation 2")],
|
|
target_lang="fr",
|
|
target_lang_name="French",
|
|
))
|
|
|
|
def test_all_pass_returns_pass(self):
|
|
result = self._run_judge_batch([
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "good"},
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "good"},
|
|
])
|
|
assert result.verdict == "pass"
|
|
assert result.chunks_passed == 2
|
|
assert result.chunks_failed == 0
|
|
assert result.failure_rate == 0.0
|
|
|
|
def test_any_fail_returns_fail(self):
|
|
result = self._run_judge_batch([
|
|
{"accurate": "yes", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "good"},
|
|
{"accurate": "no", "fluent": "yes", "correct_language": "yes",
|
|
"no_leaks": "yes", "reason": "mistranslation"},
|
|
])
|
|
assert result.verdict == "fail"
|
|
assert result.chunks_failed == 1
|
|
assert result.failure_rate == 0.5
|
|
|
|
def test_empty_pairs_returns_skip(self):
|
|
result = asyncio.run(self.judge.judge_batch([], "fr", "French"))
|
|
assert result.verdict == "skip"
|
|
assert result.chunks_evaluated == 0
|
|
|
|
def test_api_error_returns_skip(self):
|
|
# Set up a client that raises
|
|
mock_client = MagicMock()
|
|
mock_client.chat.completions.create = AsyncMock(
|
|
side_effect=Exception("API down")
|
|
)
|
|
self.judge._client = mock_client
|
|
result = asyncio.run(self.judge.judge_batch(
|
|
[("Source", "Translation")], "fr", "French"
|
|
))
|
|
assert result.verdict == "skip"
|
|
assert "API down" in result.error
|
|
|
|
def test_timeout_returns_skip(self):
|
|
import asyncio
|
|
mock_client = MagicMock()
|
|
async def slow_call(*args, **kwargs):
|
|
await asyncio.sleep(20) # longer than timeout
|
|
return MagicMock()
|
|
mock_client.chat.completions.create = slow_call
|
|
self.judge._client = mock_client
|
|
self.judge._timeout = 0.1 # very short timeout
|
|
result = asyncio.run(self.judge.judge_batch(
|
|
[("Source", "Translation")], "fr", "French"
|
|
))
|
|
assert result.verdict == "skip"
|
|
assert "timeout" in result.error
|
|
|
|
|
|
# ---------- make_judge_from_env ----------
|
|
|
|
class TestMakeJudgeFromEnv:
|
|
def test_returns_none_when_no_api_key(self, monkeypatch):
|
|
monkeypatch.delenv("L1_JUDGE_API_KEY", raising=False)
|
|
assert make_judge_from_env() is None
|
|
|
|
def test_returns_judge_when_key_set(self, monkeypatch):
|
|
monkeypatch.setenv("L1_JUDGE_API_KEY", "test-key")
|
|
monkeypatch.setenv("L1_JUDGE_MODEL", "gpt-4o-mini")
|
|
monkeypatch.setenv("L1_JUDGE_BASE_URL", "https://api.openai.com/v1")
|
|
judge = make_judge_from_env()
|
|
assert judge is not None
|
|
assert judge._model == "gpt-4o-mini"
|
|
assert judge._base_url == "https://api.openai.com/v1"
|
|
|
|
def test_default_model(self, monkeypatch):
|
|
monkeypatch.setenv("L1_JUDGE_API_KEY", "k")
|
|
monkeypatch.delenv("L1_JUDGE_MODEL", raising=False)
|
|
judge = make_judge_from_env()
|
|
assert judge._model == "deepseek-chat"
|