sepehr
c794eff823
Some checks failed
Deploy to Production / Build and Deploy (push) Has been cancelled
495 lines
18 KiB
Python
495 lines
18 KiB
Python
"""
|
||
Tests for Track A4 — L2 Pro premium judge.
|
||
|
||
Covers:
|
||
- L2DimensionVerdict: 8 dimensions + scoring
|
||
- L2Result: aggregation + dimension pass rates
|
||
- L2ProJudge: construction, missing api_key, cost estimation
|
||
- make_l2_judge_from_env: env-var-driven factory
|
||
- run_l2_check: defensive wrapper
|
||
"""
|
||
import os
|
||
import json
|
||
import pytest
|
||
from unittest.mock import MagicMock, AsyncMock, patch
|
||
|
||
from services.quality.l2_judge import (
|
||
L2DimensionVerdict,
|
||
L2Result,
|
||
L2ProJudge,
|
||
make_l2_judge_from_env,
|
||
L2_JUDGE_SYSTEM_PROMPT,
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# L2DimensionVerdict
|
||
# ============================================================================
|
||
|
||
class TestL2DimensionVerdict:
|
||
def test_all_pass(self):
|
||
v = L2DimensionVerdict(
|
||
accurate=True, fluent=True, correct_lang=True, no_leaks=True,
|
||
terminology=True, style=True, completeness=True, formatting=True,
|
||
)
|
||
assert v.passed_count == 8
|
||
assert v.total == 8
|
||
assert v.score == 1.0
|
||
assert v.passed is True
|
||
|
||
def test_one_fails(self):
|
||
v = L2DimensionVerdict(
|
||
accurate=True, fluent=True, correct_lang=True, no_leaks=True,
|
||
terminology=False, # <-- fails
|
||
style=True, completeness=True, formatting=True,
|
||
)
|
||
assert v.passed_count == 7
|
||
assert v.total == 8
|
||
assert v.score == pytest.approx(0.875)
|
||
# L2 is strict: one fail = chunk fails
|
||
assert v.passed is False
|
||
|
||
def test_all_fail(self):
|
||
v = L2DimensionVerdict() # all default False
|
||
assert v.passed_count == 0
|
||
assert v.score == 0.0
|
||
assert v.passed is False
|
||
|
||
def test_default_construction(self):
|
||
v = L2DimensionVerdict()
|
||
assert v.accurate is False
|
||
assert v.reason == ""
|
||
|
||
|
||
# ============================================================================
|
||
# L2Result
|
||
# ============================================================================
|
||
|
||
class TestL2Result:
|
||
def test_default_construction(self):
|
||
r = L2Result(verdict="pass")
|
||
assert r.verdict == "pass"
|
||
assert r.chunks_evaluated == 0
|
||
assert r.dimension_pass_rates == {}
|
||
assert r.error == ""
|
||
|
||
def test_to_log_dict(self):
|
||
r = L2Result(
|
||
verdict="pass",
|
||
chunks_evaluated=10,
|
||
chunks_passed=8,
|
||
chunks_failed=2,
|
||
failure_rate=0.2,
|
||
average_score=0.85,
|
||
model_used="gpt-4o",
|
||
cost_estimate_usd=0.012,
|
||
)
|
||
d = r.to_log_dict()
|
||
assert d["verdict"] == "pass"
|
||
assert d["chunks_evaluated"] == 10
|
||
assert d["model_used"] == "gpt-4o"
|
||
assert d["cost_estimate_usd"] == 0.012
|
||
|
||
|
||
# ============================================================================
|
||
# L2ProJudge construction
|
||
# ============================================================================
|
||
|
||
class TestL2ProJudgeConstruction:
|
||
def test_requires_api_key(self):
|
||
with pytest.raises(ValueError, match="api_key is required"):
|
||
L2ProJudge(api_key="")
|
||
|
||
def test_basic_construction(self):
|
||
judge = L2ProJudge(api_key="sk-test")
|
||
assert judge._api_key == "sk-test"
|
||
assert judge._model == "gpt-4o"
|
||
assert judge._base_url == "https://api.openai.com/v1"
|
||
|
||
def test_custom_model(self):
|
||
judge = L2ProJudge(
|
||
api_key="sk-test",
|
||
model="gpt-4o-mini",
|
||
base_url="https://api.openai.com/v1",
|
||
)
|
||
assert judge._model == "gpt-4o-mini"
|
||
|
||
def test_strips_trailing_slash_from_base_url(self):
|
||
judge = L2ProJudge(
|
||
api_key="sk-test",
|
||
base_url="https://api.example.com/v1/",
|
||
)
|
||
assert judge._base_url == "https://api.example.com/v1"
|
||
|
||
|
||
# ============================================================================
|
||
# Cost estimation
|
||
# ============================================================================
|
||
|
||
class TestL2CostEstimation:
|
||
def test_gpt4o_cost(self):
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
cost = judge._estimate_cost(15) # 15 samples default
|
||
# Should be in the $0.01–$0.05 range for 15 chunks
|
||
assert 0.001 < cost < 0.10
|
||
|
||
def test_gpt4o_mini_cheaper(self):
|
||
judge_mini = L2ProJudge(api_key="sk", model="gpt-4o-mini")
|
||
judge_full = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
cost_mini = judge_mini._estimate_cost(15)
|
||
cost_full = judge_full._estimate_cost(15)
|
||
# gpt-4o-mini should be cheaper than gpt-4o
|
||
assert cost_mini < cost_full
|
||
|
||
def test_zero_pairs(self):
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
cost = judge._estimate_cost(0)
|
||
# Even with 0 pairs, the system prompt has some cost
|
||
assert cost >= 0
|
||
|
||
|
||
# ============================================================================
|
||
# judge_batch — defensive
|
||
# ============================================================================
|
||
|
||
class TestL2JudgeBatch:
|
||
@pytest.mark.asyncio
|
||
async def test_empty_pairs_skips(self):
|
||
judge = L2ProJudge(api_key="sk")
|
||
result = await judge.judge_batch([], "fr", "French")
|
||
assert result.verdict == "skip"
|
||
assert result.error == "empty pairs"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_client_unavailable_skips(self):
|
||
judge = L2ProJudge(api_key="sk")
|
||
# Simulate a client init failure
|
||
with patch.object(judge, "_get_client", return_value=None):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
assert result.verdict == "skip"
|
||
assert "unavailable" in result.error or "client" in result.error.lower()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_successful_judgement(self):
|
||
"""A mock client that returns a well-formed JSON response should
|
||
produce a passing L2Result."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
# Mock the client
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
mock_response.choices[0].message.content = json.dumps([
|
||
{
|
||
"accurate": "yes", "fluent": "yes", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "Perfect translation",
|
||
}
|
||
])
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
|
||
assert result.verdict == "pass"
|
||
assert result.chunks_evaluated == 1
|
||
assert result.chunks_passed == 1
|
||
assert result.chunks_failed == 0
|
||
assert result.failure_rate == 0.0
|
||
assert result.average_score == 1.0
|
||
# All 8 dimensions should have pass rate 1.0
|
||
for dim in ["accurate", "fluent", "terminology", "style"]:
|
||
assert result.dimension_pass_rates[dim] == 1.0
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_partial_failure(self):
|
||
"""If one of 8 dimensions fails, the chunk should fail."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
# fluent = no, others yes
|
||
mock_response.choices[0].message.content = json.dumps([
|
||
{
|
||
"accurate": "yes", "fluent": "no", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "Awkward phrasing",
|
||
}
|
||
])
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
|
||
# L2 is strict: one fail = overall fail
|
||
assert result.verdict == "fail"
|
||
assert result.chunks_evaluated == 1
|
||
assert result.chunks_passed == 0
|
||
assert result.chunks_failed == 1
|
||
assert result.dimension_pass_rates["fluent"] == 0.0
|
||
assert result.dimension_pass_rates["accurate"] == 1.0
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_handles_markdown_fences(self):
|
||
"""The judge should strip markdown code fences from responses."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
mock_response.choices[0].message.content = (
|
||
"```json\n"
|
||
+ json.dumps([{
|
||
"accurate": "yes", "fluent": "yes", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "ok",
|
||
}])
|
||
+ "\n```"
|
||
)
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
|
||
assert result.verdict == "pass"
|
||
assert result.chunks_evaluated == 1
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_handles_dict_with_list(self):
|
||
"""Some LLMs return {"verdicts": [...]} instead of a raw list."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
mock_response.choices[0].message.content = json.dumps({
|
||
"verdicts": [
|
||
{
|
||
"accurate": "yes", "fluent": "yes", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "good",
|
||
}
|
||
]
|
||
})
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
|
||
assert result.verdict == "pass"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_timeout_returns_skip(self):
|
||
import asyncio
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o", timeout_seconds=0.1)
|
||
|
||
# Mock client that raises TimeoutError
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(
|
||
side_effect=asyncio.TimeoutError()
|
||
)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
|
||
assert result.verdict == "skip"
|
||
assert "timeout" in result.error.lower()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_never_raises(self):
|
||
"""Even on unexpected error, the judge should return a skip, not raise."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
# Mock client that raises a generic exception
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(
|
||
side_effect=RuntimeError("something unexpected")
|
||
)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
# Should NOT raise
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour")], "fr", "French"
|
||
)
|
||
assert result.verdict == "skip"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_dimension_pass_rates_aggregated(self):
|
||
"""Per-dimension pass rates should aggregate across chunks."""
|
||
judge = L2ProJudge(api_key="sk", model="gpt-4o")
|
||
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
# 2 chunks: chunk 1 all-pass, chunk 2 fluent-fail
|
||
mock_response.choices[0].message.content = json.dumps([
|
||
{
|
||
"accurate": "yes", "fluent": "yes", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "ok",
|
||
},
|
||
{
|
||
"accurate": "yes", "fluent": "no", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "awkward",
|
||
},
|
||
])
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
|
||
with patch.object(judge, "_get_client", return_value=mock_client):
|
||
result = await judge.judge_batch(
|
||
[("Hello", "Bonjour"), ("Goodbye", "Au revoir")],
|
||
"fr", "French"
|
||
)
|
||
|
||
assert result.chunks_evaluated == 2
|
||
# fluent: 1/2 = 0.5
|
||
assert result.dimension_pass_rates["fluent"] == 0.5
|
||
# accurate: 2/2 = 1.0
|
||
assert result.dimension_pass_rates["accurate"] == 1.0
|
||
|
||
|
||
# ============================================================================
|
||
# Factory
|
||
# ============================================================================
|
||
|
||
class TestL2JudgeFactory:
|
||
def test_no_api_key_returns_none(self, monkeypatch):
|
||
monkeypatch.delenv("L2_JUDGE_API_KEY", raising=False)
|
||
assert make_l2_judge_from_env() is None
|
||
|
||
def test_api_key_creates_judge(self, monkeypatch):
|
||
monkeypatch.setenv("L2_JUDGE_API_KEY", "sk-test")
|
||
monkeypatch.setenv("L2_JUDGE_MODEL", "gpt-4o")
|
||
monkeypatch.setenv("L2_JUDGE_BASE_URL", "https://api.openai.com/v1")
|
||
judge = make_l2_judge_from_env()
|
||
assert judge is not None
|
||
assert judge._api_key == "sk-test"
|
||
assert judge._model == "gpt-4o"
|
||
|
||
def test_api_key_with_default_model(self, monkeypatch):
|
||
monkeypatch.setenv("L2_JUDGE_API_KEY", "sk-test")
|
||
# Clear other vars to test defaults
|
||
monkeypatch.delenv("L2_JUDGE_MODEL", raising=False)
|
||
monkeypatch.delenv("L2_JUDGE_BASE_URL", raising=False)
|
||
judge = make_l2_judge_from_env()
|
||
assert judge._model == "gpt-4o"
|
||
assert judge._base_url == "https://api.openai.com/v1"
|
||
|
||
|
||
# ============================================================================
|
||
# Pipeline integration
|
||
# ============================================================================
|
||
|
||
class TestL2PipelineIntegration:
|
||
@pytest.mark.asyncio
|
||
async def test_run_l2_check_no_judge_skips(self):
|
||
from services.quality.pipeline import run_l2_check
|
||
|
||
result = await run_l2_check(
|
||
source_chunks=["Hello"] * 25,
|
||
translated_chunks=["Bonjour"] * 25,
|
||
target_lang="fr",
|
||
file_extension="docx",
|
||
judge=None, # Will try to load from env, but env has no key
|
||
)
|
||
# Should return skip, not raise
|
||
assert result.verdict == "skip"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_l2_check_with_mock_judge(self):
|
||
from services.quality.pipeline import run_l2_check
|
||
|
||
# Build a judge that always returns pass
|
||
judge = L2ProJudge(api_key="sk")
|
||
|
||
mock_response = MagicMock()
|
||
mock_response.choices = [MagicMock()]
|
||
mock_response.choices[0].message.content = json.dumps([
|
||
{
|
||
"accurate": "yes", "fluent": "yes", "correct_lang": "yes",
|
||
"no_leaks": "yes", "terminology": "yes", "style": "yes",
|
||
"completeness": "yes", "formatting": "yes",
|
||
"reason": "ok",
|
||
}
|
||
] * 5)
|
||
|
||
mock_client = MagicMock()
|
||
mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
|
||
judge._client = mock_client
|
||
|
||
# 25 chunks > default min_chunks of 20
|
||
result = await run_l2_check(
|
||
source_chunks=["Hello world"] * 25,
|
||
translated_chunks=["Bonjour le monde"] * 25,
|
||
target_lang="fr",
|
||
file_extension="docx",
|
||
judge=judge,
|
||
max_samples=5,
|
||
)
|
||
|
||
assert result.verdict == "pass"
|
||
assert result.chunks_evaluated >= 1
|
||
assert result.chunks_passed >= 1
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_l2_check_too_few_chunks_skips(self):
|
||
from services.quality.pipeline import run_l2_check
|
||
|
||
# 5 chunks < default min_chunks of 20
|
||
result = await run_l2_check(
|
||
source_chunks=["a"] * 5,
|
||
translated_chunks=["b"] * 5,
|
||
target_lang="fr",
|
||
min_chunks=20,
|
||
)
|
||
# Should skip due to insufficient chunks
|
||
assert result.verdict == "skip"
|
||
|
||
|
||
# ============================================================================
|
||
# 8-dimension coverage
|
||
# ============================================================================
|
||
|
||
class TestL28DimensionCoverage:
|
||
"""Sanity check: the L2 prompt template actually mentions all 8 dimensions."""
|
||
|
||
def test_prompt_has_all_8_dimensions(self):
|
||
for dim in [
|
||
"ACCURATE", "FLUENT", "CORRECT_LANG", "NO_LEAKS",
|
||
"TERMINOLOGY", "STYLE", "COMPLETENESS", "FORMATTING",
|
||
]:
|
||
assert dim in L2_JUDGE_SYSTEM_PROMPT, (
|
||
f"Dimension {dim!r} missing from L2 prompt template"
|
||
)
|
||
|
||
def test_prompt_has_format_hint(self):
|
||
# Should tell the model to respond with a JSON array
|
||
assert "JSON" in L2_JUDGE_SYSTEM_PROMPT
|
||
assert "yes" in L2_JUDGE_SYSTEM_PROMPT
|
||
assert "no" in L2_JUDGE_SYSTEM_PROMPT
|