feat(quality): A3 — L1 LLM judge via API (5 chunks, 0.0003 USD/job)

L1 quality layer — uses a cheap LLM via the OpenAI-compatible API to
validate translation quality. Designed to be the SECOND line of defense
after L0 (script detection, length, pattern).

Architecture:
  - sampler.py — picks 5 representative chunks per job (longest first,
    skips L0-failed indices, skips too-short or identical pairs)
  - llm_judge.py — OpenAI-compatible client, binary verdict per chunk
    (accurate / fluent / correct_language / no_leaks), JSON output,
    hard timeout, defensive (never raises), cost estimation built in
  - pipeline.py — defensive wrapper that integrates both, never breaks
    a translation job, always logs a structured event

Integration:
  - 5 feature flags in config.py (QUALITY_L1_ENABLED, _LOG_ONLY, etc.)
  - QUALITY_L1_LOG_ONLY=true by default: log-only mode, verdict NEVER
    blocks or retries a job
  - Reuses the chunks extracted by L0 (no double work)
  - Passes the set of L0-failed indices so L1 doesn't re-judge them
  - Wrapped in try/except so a misconfigured L1 NEVER breaks a job

Default config: deepseek-chat via DeepSeek API
  - Cost: ~0.0003 USD per job (5 chunks)
  - Speed: typically 1-2s per call, hard ceiling at 8s
  - Easy to swap: just set L1_JUDGE_BASE_URL and L1_JUDGE_MODEL

LLM judge is intentionally a SEPARATE model from the translator
(self-evaluation bias mitigation — Meta/Stanford papers 2024-2025).

Tests:
  test_sampler.py — 9 tests covering the sampling strategy
  test_llm_judge.py — 22 tests covering init, parsing, mocked API,
    cost estimation, env factory
  test_l1_pipeline.py — 6 tests covering the wrapper
  Total new: 37 tests, all pass
  Grand total quality+format: 264 tests passing (0 regression)

  All 36 new tests + 111 L0 tests + 117 existing translator tests = 264

Phase 1 (observation) for 2 weeks. Then QUALITY_L1_LOG_ONLY=false
to enable auto-retry via the fallback chain.

tests/services/quality/test_l1_pipeline.py

@@ -0,0 +1,127 @@
+"""
+Tests for services/quality/pipeline.py — the L1 wrapper.
+"""
+import asyncio
+import os
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from services.quality.pipeline import run_l1_check
+from services.quality.llm_judge import L1Result, LLMJudge
+
+
+class TestRunL1Check:
+    """Test the defensive wrapper around the LLM judge."""
+
+    def test_too_few_chunks_returns_skip(self):
+        sources = ["a", "b", "c"]  # only 3 chunks
+        translations = ["x", "y", "z"]
+        result = asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            max_samples=5, min_chunks=10,
+        ))
+        assert result.verdict == "skip"
+
+    def test_no_judge_configured_returns_skip(self, monkeypatch):
+        monkeypatch.delenv("L1_JUDGE_API_KEY", raising=False)
+        sources = [f"source {i}" for i in range(15)]
+        translations = [f"translation {i}" for i in range(15)]
+        result = asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            max_samples=5, min_chunks=10,
+        ))
+        assert result.verdict == "skip"
+        assert "judge" in result.error.lower() or "configured" in result.error.lower() or result.error == "not_run"
+
+    def test_skips_l0_failures(self):
+        """Chunks that L0 already flagged should be excluded from the sample."""
+        # All chunks are long enough to be sampled. Use unique markers per
+        # index so substring checks don't have false positives (e.g.
+        # "source1" being a substring of "source10").
+        sources = [f"srcidx{i:02d} with enough length" for i in range(15)]
+        translations = [f"transidx{i:02d} avec longueur suffisante" for i in range(15)]
+
+        # Mock judge that records what it received
+        mock_judge = MagicMock(spec=LLMJudge)
+        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
+            verdict="pass", chunks_evaluated=5, chunks_passed=5,
+            chunks_failed=0, failure_rate=0.0,
+        ))
+
+        # Mark indices 01, 03, 05, 07, 09, 11, 13 as L0 failures
+        l0_failures = {1, 3, 5, 7, 9, 11, 13}
+
+        result = asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            l0_failed_indices=l0_failures,
+            max_samples=5, min_chunks=10,
+            judge=mock_judge,
+        ))
+
+        # The mock was called with 5 pairs
+        call_args = mock_judge.judge_batch.call_args
+        pairs = call_args[0][0]
+        # The pairs should NOT include any source from the L0-failed indices
+        for src, _trans in pairs:
+            for failed_idx in l0_failures:
+                marker = f"srcidx{failed_idx:02d}"
+                assert marker not in src, (
+                    f"L0-failed source {failed_idx} was sent to L1: {src}"
+                )
+
+    def test_passes_log_only_flag(self):
+        """The log_only flag should be passed through to the LLMJudge result."""
+        mock_judge = MagicMock(spec=LLMJudge)
+        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
+            verdict="pass", chunks_evaluated=1, chunks_passed=1,
+            chunks_failed=0, failure_rate=0.0,
+        ))
+
+        sources = [f"long enough source {i} " * 3 for i in range(15)]
+        translations = [f"long enough translation {i} " * 3 for i in range(15)]
+
+        asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            max_samples=5, min_chunks=10,
+            judge=mock_judge,
+            log_only=True,
+        ))
+        # log_only doesn't affect the call, just the downstream decision
+
+    def test_never_raises_on_judge_error(self):
+        """If the judge itself raises, run_l1_check should catch it."""
+        mock_judge = MagicMock(spec=LLMJudge)
+        mock_judge.judge_batch = AsyncMock(side_effect=Exception("boom"))
+
+        sources = [f"long enough source {i} " * 3 for i in range(15)]
+        translations = [f"long enough translation {i} " * 3 for i in range(15)]
+
+        # Should NOT raise
+        result = asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            max_samples=5, min_chunks=10,
+            judge=mock_judge,
+        ))
+        assert result is not None
+        # verdict is either "skip" or an error one — never crashes the call
+
+    def test_target_lang_name_for_known_lang(self):
+        """Verify the LANG_NAMES mapping is used."""
+        mock_judge = MagicMock(spec=LLMJudge)
+        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
+            verdict="pass", chunks_evaluated=1, chunks_passed=1,
+            chunks_failed=0, failure_rate=0.0,
+        ))
+
+        sources = [f"long enough source {i} " * 3 for i in range(15)]
+        translations = [f"long enough translation {i} " * 3 for i in range(15)]
+
+        asyncio.run(run_l1_check(
+            sources, translations, "fr",
+            max_samples=5, min_chunks=10,
+            judge=mock_judge,
+        ))
+        call_args = mock_judge.judge_batch.call_args
+        # target_lang_name is the 3rd positional arg
+        target_lang_name = call_args[0][2]
+        assert target_lang_name == "French"

tests/services/quality/test_llm_judge.py

@@ -0,0 +1,258 @@
+"""
+Tests for services/quality/llm_judge.py
+Uses mocks for the OpenAI client — no actual API calls.
+"""
+import asyncio
+import json
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from services.quality.llm_judge import (
+    LLMJudge,
+    L1Result,
+    L1ChunkVerdict,
+    JUDGE_SYSTEM_PROMPT,
+    make_judge_from_env,
+)
+
+
+# ---------- Judge init ----------
+
+class TestLLMJudgeInit:
+    def test_requires_api_key(self):
+        with pytest.raises(ValueError):
+            LLMJudge(api_key="")
+
+    def test_init_with_defaults(self):
+        judge = LLMJudge(api_key="test-key")
+        assert judge._api_key == "test-key"
+        assert judge._model == "deepseek-chat"
+        assert judge._base_url == "https://api.deepseek.com/v1"
+
+    def test_init_with_custom_params(self):
+        judge = LLMJudge(
+            api_key="k",
+            base_url="https://api.openai.com/v1/",
+            model="gpt-4o-mini",
+            timeout_seconds=15.0,
+        )
+        assert judge._base_url == "https://api.openai.com/v1"  # trailing slash stripped
+        assert judge._model == "gpt-4o-mini"
+        assert judge._timeout == 15.0
+
+
+# ---------- Response parsing ----------
+
+class TestParseResponse:
+    def setup_method(self):
+        self.judge = LLMJudge(api_key="test-key")
+
+    def _make_response(self, content: str):
+        response = MagicMock()
+        response.choices = [MagicMock()]
+        response.choices[0].message.content = content
+        return response
+
+    def test_parse_pure_json_array(self):
+        content = json.dumps([
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "good"}
+        ])
+        verdicts = self.judge._parse_response(self._make_response(content), 1)
+        assert len(verdicts) == 1
+        assert verdicts[0].passed is True
+        assert verdicts[0].accurate is True
+        assert verdicts[0].reason == "good"
+
+    def test_parse_json_object_with_verdicts_key(self):
+        content = json.dumps({"verdicts": [
+            {"accurate": "no", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "lost meaning"}
+        ]})
+        verdicts = self.judge._parse_response(self._make_response(content), 1)
+        assert len(verdicts) == 1
+        assert verdicts[0].passed is False
+        assert verdicts[0].accurate is False
+        assert verdicts[0].fluent is True
+        assert verdicts[0].reason == "lost meaning"
+
+    def test_parse_json_with_markdown_fences(self):
+        content = "```json\n" + json.dumps([
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "ok"}
+        ]) + "\n```"
+        verdicts = self.judge._parse_response(self._make_response(content), 1)
+        assert len(verdicts) == 1
+        assert verdicts[0].passed is True
+
+    def test_parse_invalid_json_returns_empty(self):
+        content = "not json at all"
+        verdicts = self.judge._parse_response(self._make_response(content), 1)
+        assert verdicts == []
+
+    def test_parse_partial_verdict_defaults_to_false(self):
+        content = json.dumps([{"accurate": "yes"}])  # missing other fields
+        verdicts = self.judge._parse_response(self._make_response(content), 1)
+        assert len(verdicts) == 1
+        # All fields default to False when missing
+        assert verdicts[0].passed is False
+        assert verdicts[0].accurate is True
+        assert verdicts[0].fluent is False
+        assert verdicts[0].correct_language is False
+        assert verdicts[0].no_leaks is False
+
+    def test_parse_mixed_pass_fail(self):
+        content = json.dumps([
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "good"},
+            {"accurate": "no", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "mistranslation"},
+            {"accurate": "yes", "fluent": "no", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "awkward"},
+        ])
+        verdicts = self.judge._parse_response(self._make_response(content), 3)
+        assert len(verdicts) == 3
+        assert verdicts[0].passed is True
+        assert verdicts[1].passed is False
+        assert verdicts[2].passed is False
+
+
+# ---------- L1Result ----------
+
+class TestL1Result:
+    def test_passed_property(self):
+        v = L1ChunkVerdict(accurate=True, fluent=True, correct_language=True, no_leaks=True)
+        assert v.passed is True
+
+        v_fail = L1ChunkVerdict(accurate=True, fluent=True, correct_language=True, no_leaks=False)
+        assert v_fail.passed is False
+
+
+# ---------- Cost estimation ----------
+
+class TestCostEstimation:
+    def test_deepseek_estimate(self):
+        judge = LLMJudge(api_key="k", model="deepseek-chat")
+        cost = judge._estimate_cost(5)
+        # 5 pairs: ~1450 input + 250 output tokens
+        # Cost should be tiny (< $0.01)
+        assert 0.0 < cost < 0.01
+
+    def test_gpt4o_mini_more_expensive(self):
+        judge_ds = LLMJudge(api_key="k", model="deepseek-chat")
+        judge_gpt = LLMJudge(api_key="k", model="gpt-4o-mini")
+        assert judge_gpt._estimate_cost(5) > judge_ds._estimate_cost(5)
+
+    def test_gemini_flash_cheaper(self):
+        judge_gemini = LLMJudge(api_key="k", model="gemini-2.5-flash-lite")
+        judge_gpt = LLMJudge(api_key="k", model="gpt-4o-mini")
+        assert judge_gemini._estimate_cost(5) < judge_gpt._estimate_cost(5)
+
+
+# ---------- Judge batch (mocked API) ----------
+
+class TestJudgeBatch:
+    """Tests the full judge_batch flow with a mocked OpenAI client."""
+
+    def setup_method(self):
+        self.judge = LLMJudge(api_key="test-key", timeout_seconds=5.0)
+
+    def _mock_response_with_verdicts(self, verdicts_data):
+        response = MagicMock()
+        response.choices = [MagicMock()]
+        response.choices[0].message.content = json.dumps(verdicts_data)
+        return response
+
+    def _run_judge_batch(self, verdicts_data):
+        """Helper to run judge_batch with a mocked client."""
+        # Build the mock client BEFORE calling the method
+        response = self._mock_response_with_verdicts(verdicts_data)
+        mock_client = MagicMock()
+        mock_client.chat = MagicMock()
+        mock_client.chat.completions = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=response)
+        self.judge._client = mock_client
+        return asyncio.run(self.judge.judge_batch(
+            [("Source 1", "Translation 1"), ("Source 2", "Translation 2")],
+            target_lang="fr",
+            target_lang_name="French",
+        ))
+
+    def test_all_pass_returns_pass(self):
+        result = self._run_judge_batch([
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "good"},
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "good"},
+        ])
+        assert result.verdict == "pass"
+        assert result.chunks_passed == 2
+        assert result.chunks_failed == 0
+        assert result.failure_rate == 0.0
+
+    def test_any_fail_returns_fail(self):
+        result = self._run_judge_batch([
+            {"accurate": "yes", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "good"},
+            {"accurate": "no", "fluent": "yes", "correct_language": "yes",
+             "no_leaks": "yes", "reason": "mistranslation"},
+        ])
+        assert result.verdict == "fail"
+        assert result.chunks_failed == 1
+        assert result.failure_rate == 0.5
+
+    def test_empty_pairs_returns_skip(self):
+        result = asyncio.run(self.judge.judge_batch([], "fr", "French"))
+        assert result.verdict == "skip"
+        assert result.chunks_evaluated == 0
+
+    def test_api_error_returns_skip(self):
+        # Set up a client that raises
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(
+            side_effect=Exception("API down")
+        )
+        self.judge._client = mock_client
+        result = asyncio.run(self.judge.judge_batch(
+            [("Source", "Translation")], "fr", "French"
+        ))
+        assert result.verdict == "skip"
+        assert "API down" in result.error
+
+    def test_timeout_returns_skip(self):
+        import asyncio
+        mock_client = MagicMock()
+        async def slow_call(*args, **kwargs):
+            await asyncio.sleep(20)  # longer than timeout
+            return MagicMock()
+        mock_client.chat.completions.create = slow_call
+        self.judge._client = mock_client
+        self.judge._timeout = 0.1  # very short timeout
+        result = asyncio.run(self.judge.judge_batch(
+            [("Source", "Translation")], "fr", "French"
+        ))
+        assert result.verdict == "skip"
+        assert "timeout" in result.error
+
+
+# ---------- make_judge_from_env ----------
+
+class TestMakeJudgeFromEnv:
+    def test_returns_none_when_no_api_key(self, monkeypatch):
+        monkeypatch.delenv("L1_JUDGE_API_KEY", raising=False)
+        assert make_judge_from_env() is None
+
+    def test_returns_judge_when_key_set(self, monkeypatch):
+        monkeypatch.setenv("L1_JUDGE_API_KEY", "test-key")
+        monkeypatch.setenv("L1_JUDGE_MODEL", "gpt-4o-mini")
+        monkeypatch.setenv("L1_JUDGE_BASE_URL", "https://api.openai.com/v1")
+        judge = make_judge_from_env()
+        assert judge is not None
+        assert judge._model == "gpt-4o-mini"
+        assert judge._base_url == "https://api.openai.com/v1"
+
+    def test_default_model(self, monkeypatch):
+        monkeypatch.setenv("L1_JUDGE_API_KEY", "k")
+        monkeypatch.delenv("L1_JUDGE_MODEL", raising=False)
+        judge = make_judge_from_env()
+        assert judge._model == "deepseek-chat"

tests/services/quality/test_sampler.py

@@ -0,0 +1,101 @@
+"""
+Tests for services/quality/sampler.py
+"""
+import pytest
+
+from services.quality.sampler import sample_chunks_for_l1
+
+
+class TestSampleChunksForL1:
+    def test_empty_chunks(self):
+        result = sample_chunks_for_l1([], [], set(), max_samples=5, min_chunks=10)
+        assert result == []
+
+    def test_too_few_chunks(self):
+        sources = ["Hello", "World", "Goodbye"]
+        translations = ["Bonjour", "Monde", "Au revoir"]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=5, min_chunks=10)
+        # Only 3 chunks, min_chunks=10 → no sample
+        assert result == []
+
+    def test_min_chunks_exact(self):
+        sources = ["chunk"] * 10
+        translations = ["morceau"] * 10
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=5, min_chunks=10)
+        # Exactly 10 chunks, min_chunks=10 → should sample
+        assert len(result) == 5
+
+    def test_skips_l0_failures(self):
+        sources = ["a" * 100, "b" * 200, "c" * 300, "d" * 400]
+        translations = ["x" * 100, "y" * 200, "z" * 300, "w" * 400]
+        # Mark index 0 and 2 as L0 failures
+        result = sample_chunks_for_l1(sources, translations, {0, 2},
+                                       max_samples=10, min_chunks=3)
+        sources_returned = [s for s, t in result]
+        assert sources_returned == ["b" * 200, "d" * 400]
+
+    def test_prefers_longest_chunks(self):
+        # 10 chunks of different lengths. Each chunk has a unique marker
+        # so we can verify which were picked regardless of whitespace.
+        sources = [f"src{i}_" + "x" * (i * 5) for i in range(10)]
+        translations = [f"tr{i}_" + "y" * (i * 5) for i in range(10)]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=3, min_chunks=5)
+        # The 3 longest should be picked (indices 9, 8, 7)
+        assert len(result) == 3
+        # Verify the longest chunks were picked
+        markers_returned = [s for s, t in result]
+        assert any("src9_" in s for s in markers_returned)
+        assert any("src8_" in s for s in markers_returned)
+        assert any("src7_" in s for s in markers_returned)
+        # And the shortest were NOT picked
+        assert not any("src0_" in s for s in markers_returned)
+        assert not any("src1_" in s for s in markers_returned)
+
+    def test_skips_identical_source_translation(self):
+        # Identical pairs are probably numbers, codes, brand names
+        sources = ["12345", "Hello", "WORLD"]
+        translations = ["12345", "Bonjour", "WORLD"]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=5, min_chunks=3)
+        # "12345" and "WORLD" pairs should be skipped (identical)
+        sources_returned = [s for s, t in result]
+        assert "12345" not in sources_returned
+        assert "WORLD" not in sources_returned
+        assert "Hello" in sources_returned
+
+    def test_skips_very_short_pairs(self):
+        # Very short pairs don't have enough context
+        sources = ["a", "Hello world this is a longer test sentence",
+                   "b", "Another longer sentence for testing purposes"]
+        translations = ["x", "Bonjour le monde ceci est une phrase plus longue",
+                        "y", "Une autre phrase plus longue pour tester"]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=5, min_chunks=2)
+        # The "a"/"x" and "b"/"y" pairs should be skipped
+        sources_returned = [s for s, t in result]
+        assert "a" not in sources_returned
+        assert "b" not in sources_returned
+        assert len(result) == 2
+
+    def test_respects_max_samples(self):
+        sources = [f"long source {i} " * 10 for i in range(20)]
+        translations = [f"long translation {i} " * 10 for i in range(20)]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=3, min_chunks=5)
+        assert len(result) == 3
+
+    def test_results_in_document_order(self):
+        # The function should return in original document order,
+        # not in the length-priority order. (Use .strip()-friendly data
+        # to avoid whitespace edge cases in equality.)
+        sources = ["short source 1", "long source 2 " * 20, "medium source 3 " * 10, "tiny source 4"]
+        translations = ["court 1", "longue 2 " * 20, "moyen 3 " * 10, "minuscule 4"]
+        result = sample_chunks_for_l1(sources, translations, set(),
+                                       max_samples=4, min_chunks=2)
+        # Indices in the result should be 0, 1, 2, 3 (in document order)
+        for i, (s, t) in enumerate(result):
+            assert s == sources[i].strip(), f"Source {i} mismatch: {s!r} vs {sources[i].strip()!r}"
+            assert t == translations[i].strip(), f"Translation {i} mismatch"