office_translator/tests/services/quality/test_l1_pipeline.py

"""
Tests for services/quality/pipeline.py — the L1 wrapper.
"""
import asyncio
import os
import pytest
from unittest.mock import AsyncMock, MagicMock, patch

from services.quality.pipeline import run_l1_check
from services.quality.llm_judge import L1Result, LLMJudge


class TestRunL1Check:
    """Test the defensive wrapper around the LLM judge."""

    def test_too_few_chunks_returns_skip(self):
        sources = ["a", "b", "c"]  # only 3 chunks
        translations = ["x", "y", "z"]
        result = asyncio.run(run_l1_check(
            sources, translations, "fr",
            max_samples=5, min_chunks=10,
        ))
        assert result.verdict == "skip"

    def test_no_judge_configured_returns_skip(self, monkeypatch):
        monkeypatch.delenv("L1_JUDGE_API_KEY", raising=False)
        sources = [f"source {i}" for i in range(15)]
        translations = [f"translation {i}" for i in range(15)]
        result = asyncio.run(run_l1_check(
            sources, translations, "fr",
            max_samples=5, min_chunks=10,
        ))
        assert result.verdict == "skip"
        assert "judge" in result.error.lower() or "configured" in result.error.lower() or result.error == "not_run"

    def test_skips_l0_failures(self):
        """Chunks that L0 already flagged should be excluded from the sample."""
        # All chunks are long enough to be sampled. Use unique markers per
        # index so substring checks don't have false positives (e.g.
        # "source1" being a substring of "source10").
        sources = [f"srcidx{i:02d} with enough length" for i in range(15)]
        translations = [f"transidx{i:02d} avec longueur suffisante" for i in range(15)]

        # Mock judge that records what it received
        mock_judge = MagicMock(spec=LLMJudge)
        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
            verdict="pass", chunks_evaluated=5, chunks_passed=5,
            chunks_failed=0, failure_rate=0.0,
        ))

        # Mark indices 01, 03, 05, 07, 09, 11, 13 as L0 failures
        l0_failures = {1, 3, 5, 7, 9, 11, 13}

        result = asyncio.run(run_l1_check(
            sources, translations, "fr",
            l0_failed_indices=l0_failures,
            max_samples=5, min_chunks=10,
            judge=mock_judge,
        ))

        # The mock was called with 5 pairs
        call_args = mock_judge.judge_batch.call_args
        pairs = call_args[0][0]
        # The pairs should NOT include any source from the L0-failed indices
        for src, _trans in pairs:
            for failed_idx in l0_failures:
                marker = f"srcidx{failed_idx:02d}"
                assert marker not in src, (
                    f"L0-failed source {failed_idx} was sent to L1: {src}"
                )

    def test_passes_log_only_flag(self):
        """The log_only flag should be passed through to the LLMJudge result."""
        mock_judge = MagicMock(spec=LLMJudge)
        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
            verdict="pass", chunks_evaluated=1, chunks_passed=1,
            chunks_failed=0, failure_rate=0.0,
        ))

        sources = [f"long enough source {i} " * 3 for i in range(15)]
        translations = [f"long enough translation {i} " * 3 for i in range(15)]

        asyncio.run(run_l1_check(
            sources, translations, "fr",
            max_samples=5, min_chunks=10,
            judge=mock_judge,
            log_only=True,
        ))
        # log_only doesn't affect the call, just the downstream decision

    def test_never_raises_on_judge_error(self):
        """If the judge itself raises, run_l1_check should catch it."""
        mock_judge = MagicMock(spec=LLMJudge)
        mock_judge.judge_batch = AsyncMock(side_effect=Exception("boom"))

        sources = [f"long enough source {i} " * 3 for i in range(15)]
        translations = [f"long enough translation {i} " * 3 for i in range(15)]

        # Should NOT raise
        result = asyncio.run(run_l1_check(
            sources, translations, "fr",
            max_samples=5, min_chunks=10,
            judge=mock_judge,
        ))
        assert result is not None
        # verdict is either "skip" or an error one — never crashes the call

    def test_target_lang_name_for_known_lang(self):
        """Verify the LANG_NAMES mapping is used."""
        mock_judge = MagicMock(spec=LLMJudge)
        mock_judge.judge_batch = AsyncMock(return_value=L1Result(
            verdict="pass", chunks_evaluated=1, chunks_passed=1,
            chunks_failed=0, failure_rate=0.0,
        ))

        sources = [f"long enough source {i} " * 3 for i in range(15)]
        translations = [f"long enough translation {i} " * 3 for i in range(15)]

        asyncio.run(run_l1_check(
            sources, translations, "fr",
            max_samples=5, min_chunks=10,
            judge=mock_judge,
        ))
        call_args = mock_judge.judge_batch.call_args
        # target_lang_name is the 3rd positional arg
        target_lang_name = call_args[0][2]
        assert target_lang_name == "French"