feat(quality): A4 — L2 Pro premium judge (8 dims, gpt-4o, Pro-gated, opt-in)

tests/services/quality/test_l2_judge.py

@@ -0,0 +1,494 @@
+"""
+Tests for Track A4 — L2 Pro premium judge.
+
+Covers:
+  - L2DimensionVerdict: 8 dimensions + scoring
+  - L2Result: aggregation + dimension pass rates
+  - L2ProJudge: construction, missing api_key, cost estimation
+  - make_l2_judge_from_env: env-var-driven factory
+  - run_l2_check: defensive wrapper
+"""
+import os
+import json
+import pytest
+from unittest.mock import MagicMock, AsyncMock, patch
+
+from services.quality.l2_judge import (
+    L2DimensionVerdict,
+    L2Result,
+    L2ProJudge,
+    make_l2_judge_from_env,
+    L2_JUDGE_SYSTEM_PROMPT,
+)
+
+
+# ============================================================================
+# L2DimensionVerdict
+# ============================================================================
+
+class TestL2DimensionVerdict:
+    def test_all_pass(self):
+        v = L2DimensionVerdict(
+            accurate=True, fluent=True, correct_lang=True, no_leaks=True,
+            terminology=True, style=True, completeness=True, formatting=True,
+        )
+        assert v.passed_count == 8
+        assert v.total == 8
+        assert v.score == 1.0
+        assert v.passed is True
+
+    def test_one_fails(self):
+        v = L2DimensionVerdict(
+            accurate=True, fluent=True, correct_lang=True, no_leaks=True,
+            terminology=False,  # <-- fails
+            style=True, completeness=True, formatting=True,
+        )
+        assert v.passed_count == 7
+        assert v.total == 8
+        assert v.score == pytest.approx(0.875)
+        # L2 is strict: one fail = chunk fails
+        assert v.passed is False
+
+    def test_all_fail(self):
+        v = L2DimensionVerdict()  # all default False
+        assert v.passed_count == 0
+        assert v.score == 0.0
+        assert v.passed is False
+
+    def test_default_construction(self):
+        v = L2DimensionVerdict()
+        assert v.accurate is False
+        assert v.reason == ""
+
+
+# ============================================================================
+# L2Result
+# ============================================================================
+
+class TestL2Result:
+    def test_default_construction(self):
+        r = L2Result(verdict="pass")
+        assert r.verdict == "pass"
+        assert r.chunks_evaluated == 0
+        assert r.dimension_pass_rates == {}
+        assert r.error == ""
+
+    def test_to_log_dict(self):
+        r = L2Result(
+            verdict="pass",
+            chunks_evaluated=10,
+            chunks_passed=8,
+            chunks_failed=2,
+            failure_rate=0.2,
+            average_score=0.85,
+            model_used="gpt-4o",
+            cost_estimate_usd=0.012,
+        )
+        d = r.to_log_dict()
+        assert d["verdict"] == "pass"
+        assert d["chunks_evaluated"] == 10
+        assert d["model_used"] == "gpt-4o"
+        assert d["cost_estimate_usd"] == 0.012
+
+
+# ============================================================================
+# L2ProJudge construction
+# ============================================================================
+
+class TestL2ProJudgeConstruction:
+    def test_requires_api_key(self):
+        with pytest.raises(ValueError, match="api_key is required"):
+            L2ProJudge(api_key="")
+
+    def test_basic_construction(self):
+        judge = L2ProJudge(api_key="sk-test")
+        assert judge._api_key == "sk-test"
+        assert judge._model == "gpt-4o"
+        assert judge._base_url == "https://api.openai.com/v1"
+
+    def test_custom_model(self):
+        judge = L2ProJudge(
+            api_key="sk-test",
+            model="gpt-4o-mini",
+            base_url="https://api.openai.com/v1",
+        )
+        assert judge._model == "gpt-4o-mini"
+
+    def test_strips_trailing_slash_from_base_url(self):
+        judge = L2ProJudge(
+            api_key="sk-test",
+            base_url="https://api.example.com/v1/",
+        )
+        assert judge._base_url == "https://api.example.com/v1"
+
+
+# ============================================================================
+# Cost estimation
+# ============================================================================
+
+class TestL2CostEstimation:
+    def test_gpt4o_cost(self):
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+        cost = judge._estimate_cost(15)  # 15 samples default
+        # Should be in the $0.01–$0.05 range for 15 chunks
+        assert 0.001 < cost < 0.10
+
+    def test_gpt4o_mini_cheaper(self):
+        judge_mini = L2ProJudge(api_key="sk", model="gpt-4o-mini")
+        judge_full = L2ProJudge(api_key="sk", model="gpt-4o")
+        cost_mini = judge_mini._estimate_cost(15)
+        cost_full = judge_full._estimate_cost(15)
+        # gpt-4o-mini should be cheaper than gpt-4o
+        assert cost_mini < cost_full
+
+    def test_zero_pairs(self):
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+        cost = judge._estimate_cost(0)
+        # Even with 0 pairs, the system prompt has some cost
+        assert cost >= 0
+
+
+# ============================================================================
+# judge_batch — defensive
+# ============================================================================
+
+class TestL2JudgeBatch:
+    @pytest.mark.asyncio
+    async def test_empty_pairs_skips(self):
+        judge = L2ProJudge(api_key="sk")
+        result = await judge.judge_batch([], "fr", "French")
+        assert result.verdict == "skip"
+        assert result.error == "empty pairs"
+
+    @pytest.mark.asyncio
+    async def test_client_unavailable_skips(self):
+        judge = L2ProJudge(api_key="sk")
+        # Simulate a client init failure
+        with patch.object(judge, "_get_client", return_value=None):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+        assert result.verdict == "skip"
+        assert "unavailable" in result.error or "client" in result.error.lower()
+
+    @pytest.mark.asyncio
+    async def test_successful_judgement(self):
+        """A mock client that returns a well-formed JSON response should
+        produce a passing L2Result."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        # Mock the client
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        mock_response.choices[0].message.content = json.dumps([
+            {
+                "accurate": "yes", "fluent": "yes", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "Perfect translation",
+            }
+        ])
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+
+        assert result.verdict == "pass"
+        assert result.chunks_evaluated == 1
+        assert result.chunks_passed == 1
+        assert result.chunks_failed == 0
+        assert result.failure_rate == 0.0
+        assert result.average_score == 1.0
+        # All 8 dimensions should have pass rate 1.0
+        for dim in ["accurate", "fluent", "terminology", "style"]:
+            assert result.dimension_pass_rates[dim] == 1.0
+
+    @pytest.mark.asyncio
+    async def test_partial_failure(self):
+        """If one of 8 dimensions fails, the chunk should fail."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        # fluent = no, others yes
+        mock_response.choices[0].message.content = json.dumps([
+            {
+                "accurate": "yes", "fluent": "no", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "Awkward phrasing",
+            }
+        ])
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+
+        # L2 is strict: one fail = overall fail
+        assert result.verdict == "fail"
+        assert result.chunks_evaluated == 1
+        assert result.chunks_passed == 0
+        assert result.chunks_failed == 1
+        assert result.dimension_pass_rates["fluent"] == 0.0
+        assert result.dimension_pass_rates["accurate"] == 1.0
+
+    @pytest.mark.asyncio
+    async def test_handles_markdown_fences(self):
+        """The judge should strip markdown code fences from responses."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        mock_response.choices[0].message.content = (
+            "```json\n"
+            + json.dumps([{
+                "accurate": "yes", "fluent": "yes", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "ok",
+            }])
+            + "\n```"
+        )
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+
+        assert result.verdict == "pass"
+        assert result.chunks_evaluated == 1
+
+    @pytest.mark.asyncio
+    async def test_handles_dict_with_list(self):
+        """Some LLMs return {"verdicts": [...]} instead of a raw list."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        mock_response.choices[0].message.content = json.dumps({
+            "verdicts": [
+                {
+                    "accurate": "yes", "fluent": "yes", "correct_lang": "yes",
+                    "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                    "completeness": "yes", "formatting": "yes",
+                    "reason": "good",
+                }
+            ]
+        })
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+
+        assert result.verdict == "pass"
+
+    @pytest.mark.asyncio
+    async def test_timeout_returns_skip(self):
+        import asyncio
+        judge = L2ProJudge(api_key="sk", model="gpt-4o", timeout_seconds=0.1)
+
+        # Mock client that raises TimeoutError
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(
+            side_effect=asyncio.TimeoutError()
+        )
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+
+        assert result.verdict == "skip"
+        assert "timeout" in result.error.lower()
+
+    @pytest.mark.asyncio
+    async def test_never_raises(self):
+        """Even on unexpected error, the judge should return a skip, not raise."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        # Mock client that raises a generic exception
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(
+            side_effect=RuntimeError("something unexpected")
+        )
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            # Should NOT raise
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour")], "fr", "French"
+            )
+        assert result.verdict == "skip"
+
+    @pytest.mark.asyncio
+    async def test_dimension_pass_rates_aggregated(self):
+        """Per-dimension pass rates should aggregate across chunks."""
+        judge = L2ProJudge(api_key="sk", model="gpt-4o")
+
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        # 2 chunks: chunk 1 all-pass, chunk 2 fluent-fail
+        mock_response.choices[0].message.content = json.dumps([
+            {
+                "accurate": "yes", "fluent": "yes", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "ok",
+            },
+            {
+                "accurate": "yes", "fluent": "no", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "awkward",
+            },
+        ])
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+
+        with patch.object(judge, "_get_client", return_value=mock_client):
+            result = await judge.judge_batch(
+                [("Hello", "Bonjour"), ("Goodbye", "Au revoir")],
+                "fr", "French"
+            )
+
+        assert result.chunks_evaluated == 2
+        # fluent: 1/2 = 0.5
+        assert result.dimension_pass_rates["fluent"] == 0.5
+        # accurate: 2/2 = 1.0
+        assert result.dimension_pass_rates["accurate"] == 1.0
+
+
+# ============================================================================
+# Factory
+# ============================================================================
+
+class TestL2JudgeFactory:
+    def test_no_api_key_returns_none(self, monkeypatch):
+        monkeypatch.delenv("L2_JUDGE_API_KEY", raising=False)
+        assert make_l2_judge_from_env() is None
+
+    def test_api_key_creates_judge(self, monkeypatch):
+        monkeypatch.setenv("L2_JUDGE_API_KEY", "sk-test")
+        monkeypatch.setenv("L2_JUDGE_MODEL", "gpt-4o")
+        monkeypatch.setenv("L2_JUDGE_BASE_URL", "https://api.openai.com/v1")
+        judge = make_l2_judge_from_env()
+        assert judge is not None
+        assert judge._api_key == "sk-test"
+        assert judge._model == "gpt-4o"
+
+    def test_api_key_with_default_model(self, monkeypatch):
+        monkeypatch.setenv("L2_JUDGE_API_KEY", "sk-test")
+        # Clear other vars to test defaults
+        monkeypatch.delenv("L2_JUDGE_MODEL", raising=False)
+        monkeypatch.delenv("L2_JUDGE_BASE_URL", raising=False)
+        judge = make_l2_judge_from_env()
+        assert judge._model == "gpt-4o"
+        assert judge._base_url == "https://api.openai.com/v1"
+
+
+# ============================================================================
+# Pipeline integration
+# ============================================================================
+
+class TestL2PipelineIntegration:
+    @pytest.mark.asyncio
+    async def test_run_l2_check_no_judge_skips(self):
+        from services.quality.pipeline import run_l2_check
+
+        result = await run_l2_check(
+            source_chunks=["Hello"] * 25,
+            translated_chunks=["Bonjour"] * 25,
+            target_lang="fr",
+            file_extension="docx",
+            judge=None,  # Will try to load from env, but env has no key
+        )
+        # Should return skip, not raise
+        assert result.verdict == "skip"
+
+    @pytest.mark.asyncio
+    async def test_run_l2_check_with_mock_judge(self):
+        from services.quality.pipeline import run_l2_check
+
+        # Build a judge that always returns pass
+        judge = L2ProJudge(api_key="sk")
+
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        mock_response.choices[0].message.content = json.dumps([
+            {
+                "accurate": "yes", "fluent": "yes", "correct_lang": "yes",
+                "no_leaks": "yes", "terminology": "yes", "style": "yes",
+                "completeness": "yes", "formatting": "yes",
+                "reason": "ok",
+            }
+        ] * 5)
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+        judge._client = mock_client
+
+        # 25 chunks > default min_chunks of 20
+        result = await run_l2_check(
+            source_chunks=["Hello world"] * 25,
+            translated_chunks=["Bonjour le monde"] * 25,
+            target_lang="fr",
+            file_extension="docx",
+            judge=judge,
+            max_samples=5,
+        )
+
+        assert result.verdict == "pass"
+        assert result.chunks_evaluated >= 1
+        assert result.chunks_passed >= 1
+
+    @pytest.mark.asyncio
+    async def test_run_l2_check_too_few_chunks_skips(self):
+        from services.quality.pipeline import run_l2_check
+
+        # 5 chunks < default min_chunks of 20
+        result = await run_l2_check(
+            source_chunks=["a"] * 5,
+            translated_chunks=["b"] * 5,
+            target_lang="fr",
+            min_chunks=20,
+        )
+        # Should skip due to insufficient chunks
+        assert result.verdict == "skip"
+
+
+# ============================================================================
+# 8-dimension coverage
+# ============================================================================
+
+class TestL28DimensionCoverage:
+    """Sanity check: the L2 prompt template actually mentions all 8 dimensions."""
+
+    def test_prompt_has_all_8_dimensions(self):
+        for dim in [
+            "ACCURATE", "FLUENT", "CORRECT_LANG", "NO_LEAKS",
+            "TERMINOLOGY", "STYLE", "COMPLETENESS", "FORMATTING",
+        ]:
+            assert dim in L2_JUDGE_SYSTEM_PROMPT, (
+                f"Dimension {dim!r} missing from L2 prompt template"
+            )
+
+    def test_prompt_has_format_hint(self):
+        # Should tell the model to respond with a JSON array
+        assert "JSON" in L2_JUDGE_SYSTEM_PROMPT
+        assert "yes" in L2_JUDGE_SYSTEM_PROMPT
+        assert "no" in L2_JUDGE_SYSTEM_PROMPT