sepehr
c794eff823
Some checks failed
Deploy to Production / Build and Deploy (push) Has been cancelled
396 lines
15 KiB
Python
396 lines
15 KiB
Python
"""
|
|
L2 Pro Premium Judge — stronger model, more dimensions, Pro-tier only.
|
|
|
|
Why a SEPARATE module from L1?
|
|
- L1 is fast + cheap (deepseek-chat, 4 dimensions, 5 samples)
|
|
- L2 is slow + expensive (gpt-4o, 8 dimensions, 15 samples)
|
|
- Different defaults, different config, different metrics
|
|
- L2 is gated to the Pro plan; L1 is universal
|
|
|
|
L2 dimensions (8):
|
|
1. accurate — meaning preserved
|
|
2. fluent — natural in target language
|
|
3. correct_lang — in the target language (not source leakage)
|
|
4. no_leaks — no prompt artifacts
|
|
5. terminology — domain terms correctly handled
|
|
6. style — appropriate register (formal/informal/technical)
|
|
7. completeness — no content dropped or added
|
|
8. formatting — codes, numbers, units preserved
|
|
|
|
L1 was a binary pass/fail. L2 returns per-dimension scores (0/1) so the
|
|
caller can decide which dimensions matter for a given job.
|
|
|
|
DESIGN CONSTRAINTS (same as L1):
|
|
- 100% API-based. No local models, no GPU.
|
|
- Async, with a hard timeout.
|
|
- Defensive: never raises.
|
|
- Output is structured JSON.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field, asdict
|
|
from typing import List, Optional, Tuple, Dict, Any
|
|
|
|
from core.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
# ---------- Result dataclasses ----------
|
|
|
|
@dataclass
|
|
class L2DimensionVerdict:
|
|
"""8-dimension verdict for a single chunk."""
|
|
accurate: bool = False
|
|
fluent: bool = False
|
|
correct_lang: bool = False
|
|
no_leaks: bool = False
|
|
terminology: bool = False
|
|
style: bool = False
|
|
completeness: bool = False
|
|
formatting: bool = False
|
|
reason: str = ""
|
|
|
|
@property
|
|
def passed_count(self) -> int:
|
|
return sum([
|
|
self.accurate, self.fluent, self.correct_lang, self.no_leaks,
|
|
self.terminology, self.style, self.completeness, self.formatting,
|
|
])
|
|
|
|
@property
|
|
def total(self) -> int:
|
|
return 8
|
|
|
|
@property
|
|
def score(self) -> float:
|
|
return self.passed_count / self.total
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
# L2 is conservative: any single fail = chunk fails
|
|
return self.passed_count == self.total
|
|
|
|
def to_log_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass
|
|
class L2Result:
|
|
"""Aggregate result of an L2 check on a sample of chunks."""
|
|
verdict: str # "pass", "fail", "skip"
|
|
chunks_evaluated: int = 0
|
|
chunks_passed: int = 0
|
|
chunks_failed: int = 0
|
|
failure_rate: float = 0.0
|
|
average_score: float = 0.0 # mean of per-chunk scores (0.0 to 1.0)
|
|
dimension_pass_rates: Dict[str, float] = field(default_factory=dict)
|
|
samples: List[dict] = field(default_factory=list)
|
|
model_used: str = ""
|
|
elapsed_ms: float = 0.0
|
|
cost_estimate_usd: float = 0.0
|
|
error: str = ""
|
|
|
|
def to_log_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
|
|
# ---------- Prompt template (8 dimensions) ----------
|
|
|
|
L2_JUDGE_SYSTEM_PROMPT = """You are an expert translation quality evaluator using MQM-inspired criteria.
|
|
|
|
For each (SOURCE, TRANSLATION) pair, check these 8 criteria (yes/no for each):
|
|
|
|
1. ACCURATE — Does the translation preserve the meaning of the source?
|
|
2. FLUENT — Is the translation natural and grammatical in {target_lang_name}?
|
|
3. CORRECT_LANG — Is the translation actually in {target_lang_name} (ISO: {target_lang})?
|
|
4. NO_LEAKS — Is the translation free of prompt artifacts, source-language text, or meta-commentary?
|
|
5. TERMINOLOGY — Are domain-specific terms (technical, legal, medical, etc.) correctly translated?
|
|
6. STYLE — Is the register/tone appropriate (formal/informal/technical matching the source)?
|
|
7. COMPLETENESS — Is all content present, with nothing added or dropped?
|
|
8. FORMATTING — Are codes, numbers, dates, and units preserved exactly?
|
|
|
|
A translation FAILS if ANY criterion is "no". The "reason" must be in English and ≤ 20 words.
|
|
|
|
Respond with a JSON array, one object per pair, in the same order. NO other text, NO markdown fences:
|
|
[
|
|
{{"accurate": "yes"|"no", "fluent": "yes"|"no", "correct_lang": "yes"|"no", "no_leaks": "yes"|"no", "terminology": "yes"|"no", "style": "yes"|"no", "completeness": "yes"|"no", "formatting": "yes"|"no", "reason": "short justification"}}
|
|
]
|
|
"""
|
|
|
|
|
|
# ---------- LLM client ----------
|
|
|
|
class L2ProJudge:
|
|
"""
|
|
Calls a STRONG LLM via the OpenAI-compatible API to judge translation
|
|
quality across 8 dimensions. Pro-tier only.
|
|
|
|
Default model: gpt-4o (strongest general judge we can afford at scale).
|
|
For sub-$0.01/job cost, we limit to 15 samples per job and 8 dimensions.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
api_key: str,
|
|
base_url: str = "https://api.openai.com/v1",
|
|
model: str = "gpt-4o",
|
|
timeout_seconds: float = 20.0,
|
|
max_retries: int = 1,
|
|
):
|
|
if not api_key:
|
|
raise ValueError("api_key is required for L2ProJudge")
|
|
self._api_key = api_key
|
|
self._base_url = base_url.rstrip("/")
|
|
self._model = model
|
|
self._timeout = timeout_seconds
|
|
self._max_retries = max_retries
|
|
self._client = None
|
|
|
|
def _get_client(self):
|
|
if self._client is None:
|
|
try:
|
|
import openai
|
|
self._client = openai.AsyncOpenAI(
|
|
api_key=self._api_key,
|
|
base_url=self._base_url,
|
|
timeout=self._timeout,
|
|
)
|
|
except Exception as e:
|
|
logger.warning("l2_judge_client_init_failed", error=str(e))
|
|
return None
|
|
return self._client
|
|
|
|
async def judge_batch(
|
|
self,
|
|
pairs: List[Tuple[str, str]],
|
|
target_lang: str,
|
|
target_lang_name: str = "",
|
|
) -> L2Result:
|
|
"""
|
|
Judge a batch of (source, translation) pairs across 8 dimensions.
|
|
|
|
Returns an L2Result with verdict="skip" on any internal error.
|
|
Never raises.
|
|
"""
|
|
start = time.time()
|
|
empty = L2Result(verdict="skip", error="not_run")
|
|
|
|
if not pairs:
|
|
return L2Result(verdict="skip", error="empty pairs",
|
|
elapsed_ms=round((time.time() - start) * 1000, 2))
|
|
|
|
client = self._get_client()
|
|
if client is None:
|
|
return L2Result(verdict="skip", error="client unavailable",
|
|
elapsed_ms=round((time.time() - start) * 1000, 2))
|
|
|
|
user_lines = [f"Target language: {target_lang} ({target_lang_name})\n"]
|
|
for i, (src, trans) in enumerate(pairs, 1):
|
|
user_lines.append(f"\n--- Pair {i} ---")
|
|
user_lines.append(f"SOURCE: {src}")
|
|
user_lines.append(f"TRANSLATION: {trans}")
|
|
user_msg = "\n".join(user_lines)
|
|
|
|
system_prompt = L2_JUDGE_SYSTEM_PROMPT.format(
|
|
target_lang=target_lang,
|
|
target_lang_name=target_lang_name or target_lang,
|
|
)
|
|
|
|
try:
|
|
response = await asyncio.wait_for(
|
|
self._call_with_retries(client, system_prompt, user_msg),
|
|
timeout=self._timeout + 5.0,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
elapsed_ms = round((time.time() - start) * 1000, 2)
|
|
logger.warning("l2_judge_timeout",
|
|
timeout_s=self._timeout, elapsed_ms=elapsed_ms)
|
|
return L2Result(verdict="skip", error="timeout", elapsed_ms=elapsed_ms)
|
|
except Exception as e:
|
|
elapsed_ms = round((time.time() - start) * 1000, 2)
|
|
logger.warning("l2_judge_error",
|
|
error=str(e)[:200], elapsed_ms=elapsed_ms)
|
|
return L2Result(verdict="skip", error=str(e)[:200],
|
|
elapsed_ms=elapsed_ms)
|
|
|
|
verdicts = self._parse_response(response, len(pairs))
|
|
|
|
if not verdicts:
|
|
elapsed_ms = round((time.time() - start) * 1000, 2)
|
|
return L2Result(verdict="skip", error="parse_failed",
|
|
elapsed_ms=elapsed_ms)
|
|
|
|
# Aggregate
|
|
passed = sum(1 for v in verdicts if v.passed)
|
|
failed = len(verdicts) - passed
|
|
failure_rate = failed / len(verdicts) if verdicts else 0.0
|
|
average_score = sum(v.score for v in verdicts) / len(verdicts)
|
|
|
|
# Per-dimension pass rates
|
|
dimensions = [
|
|
"accurate", "fluent", "correct_lang", "no_leaks",
|
|
"terminology", "style", "completeness", "formatting",
|
|
]
|
|
dim_pass_rates = {}
|
|
for dim in dimensions:
|
|
count = sum(1 for v in verdicts if getattr(v, dim))
|
|
dim_pass_rates[dim] = round(count / len(verdicts), 3) if verdicts else 0.0
|
|
|
|
# L2 verdict: strict — any chunk fail = overall fail
|
|
verdict = "pass" if failed == 0 else "fail"
|
|
|
|
elapsed_ms = round((time.time() - start) * 1000, 2)
|
|
cost_estimate = self._estimate_cost(len(pairs))
|
|
|
|
return L2Result(
|
|
verdict=verdict,
|
|
chunks_evaluated=len(verdicts),
|
|
chunks_passed=passed,
|
|
chunks_failed=failed,
|
|
failure_rate=round(failure_rate, 3),
|
|
average_score=round(average_score, 3),
|
|
dimension_pass_rates=dim_pass_rates,
|
|
samples=[v.to_log_dict() for v in verdicts],
|
|
model_used=self._model,
|
|
elapsed_ms=elapsed_ms,
|
|
cost_estimate_usd=cost_estimate,
|
|
)
|
|
|
|
async def _call_with_retries(self, client, system_prompt: str, user_msg: str):
|
|
"""Call the LLM with retry on transient errors."""
|
|
last_exc = None
|
|
for attempt in range(self._max_retries + 1):
|
|
try:
|
|
response = await client.chat.completions.create(
|
|
model=self._model,
|
|
messages=[
|
|
{"role": "system", "content": system_prompt},
|
|
{"role": "user", "content": user_msg},
|
|
],
|
|
temperature=0.0,
|
|
max_tokens=1200, # larger than L1 (more dimensions)
|
|
response_format={"type": "json_object"},
|
|
)
|
|
return response
|
|
except Exception as e:
|
|
last_exc = e
|
|
if attempt < self._max_retries:
|
|
await asyncio.sleep(0.8)
|
|
raise last_exc
|
|
|
|
def _parse_response(self, response, expected_count: int) -> List[L2DimensionVerdict]:
|
|
"""Parse the LLM response into a list of 8-dimension verdicts."""
|
|
try:
|
|
content = response.choices[0].message.content or ""
|
|
except (AttributeError, IndexError) as e:
|
|
logger.warning("l2_judge_bad_response", error=str(e))
|
|
return []
|
|
|
|
content = content.strip()
|
|
if content.startswith("```"):
|
|
content = re.sub(r"^```(?:json)?\s*\n?", "", content)
|
|
content = re.sub(r"\n?```\s*$", "", content)
|
|
|
|
try:
|
|
data = json.loads(content)
|
|
except json.JSONDecodeError as e:
|
|
logger.warning("l2_judge_json_parse_error",
|
|
error=str(e), content_preview=content[:200])
|
|
return []
|
|
|
|
if isinstance(data, dict):
|
|
items = None
|
|
for key in ("verdicts", "results", "translations", "data"):
|
|
if key in data and isinstance(data[key], list):
|
|
items = data[key]
|
|
break
|
|
if items is None:
|
|
for v in data.values():
|
|
if isinstance(v, list):
|
|
items = v
|
|
break
|
|
if items is None:
|
|
logger.warning("l2_judge_no_list_in_response")
|
|
return []
|
|
elif isinstance(data, list):
|
|
items = data
|
|
else:
|
|
logger.warning("l2_judge_unexpected_response_type",
|
|
type_=type(data).__name__)
|
|
return []
|
|
|
|
verdicts: List[L2DimensionVerdict] = []
|
|
for item in items:
|
|
try:
|
|
v = L2DimensionVerdict(
|
|
accurate=str(item.get("accurate", "")).lower() == "yes",
|
|
fluent=str(item.get("fluent", "")).lower() == "yes",
|
|
correct_lang=str(item.get("correct_lang", "")).lower() == "yes",
|
|
no_leaks=str(item.get("no_leaks", "")).lower() == "yes",
|
|
terminology=str(item.get("terminology", "")).lower() == "yes",
|
|
style=str(item.get("style", "")).lower() == "yes",
|
|
completeness=str(item.get("completeness", "")).lower() == "yes",
|
|
formatting=str(item.get("formatting", "")).lower() == "yes",
|
|
reason=str(item.get("reason", ""))[:300],
|
|
)
|
|
verdicts.append(v)
|
|
except Exception as e:
|
|
logger.warning("l2_judge_item_parse_error",
|
|
error=str(e), item=str(item)[:200])
|
|
|
|
return verdicts
|
|
|
|
def _estimate_cost(self, num_pairs: int) -> float:
|
|
"""Rough USD cost estimate for the call."""
|
|
# L2 has more dimensions = longer output
|
|
input_tokens = 250 + (num_pairs * 280)
|
|
output_tokens = num_pairs * 110
|
|
model_lower = self._model.lower()
|
|
# IMPORTANT: check 'mini' BEFORE full 'gpt-4o' because
|
|
# 'gpt-4o-mini' contains 'gpt-4o'.
|
|
if "gpt-4o-mini" in model_lower:
|
|
input_cost = input_tokens / 1_000_000 * 0.15
|
|
output_cost = output_tokens / 1_000_000 * 0.60
|
|
elif "gpt-4o" in model_lower:
|
|
input_cost = input_tokens / 1_000_000 * 2.50
|
|
output_cost = output_tokens / 1_000_000 * 10.00
|
|
elif "claude" in model_lower:
|
|
input_cost = input_tokens / 1_000_000 * 3.00
|
|
output_cost = output_tokens / 1_000_000 * 15.00
|
|
else:
|
|
# Generic conservative
|
|
input_cost = input_tokens / 1_000_000 * 1.00
|
|
output_cost = output_tokens / 1_000_000 * 3.00
|
|
return round(input_cost + output_cost, 6)
|
|
|
|
|
|
# ---------- Convenience factory ----------
|
|
|
|
def make_l2_judge_from_env() -> Optional[L2ProJudge]:
|
|
"""
|
|
Build an L2ProJudge from environment variables. Returns None if
|
|
no API key is configured.
|
|
|
|
Reads:
|
|
- L2_JUDGE_API_KEY (required)
|
|
- L2_JUDGE_BASE_URL (default: OpenAI)
|
|
- L2_JUDGE_MODEL (default: gpt-4o)
|
|
- L2_JUDGE_TIMEOUT (default: 20.0)
|
|
"""
|
|
import os
|
|
api_key = os.getenv("L2_JUDGE_API_KEY", "").strip()
|
|
if not api_key:
|
|
return None
|
|
return L2ProJudge(
|
|
api_key=api_key,
|
|
base_url=os.getenv("L2_JUDGE_BASE_URL", "https://api.openai.com/v1"),
|
|
model=os.getenv("L2_JUDGE_MODEL", "gpt-4o"),
|
|
timeout_seconds=float(os.getenv("L2_JUDGE_TIMEOUT", "20.0")),
|
|
)
|