feat(quality): A4 — L2 Pro premium judge (8 dims, gpt-4o, Pro-gated, opt-in)

services/quality/l2_judge.py

@@ -0,0 +1,395 @@
+"""
+L2 Pro Premium Judge — stronger model, more dimensions, Pro-tier only.
+
+Why a SEPARATE module from L1?
+  - L1 is fast + cheap (deepseek-chat, 4 dimensions, 5 samples)
+  - L2 is slow + expensive (gpt-4o, 8 dimensions, 15 samples)
+  - Different defaults, different config, different metrics
+  - L2 is gated to the Pro plan; L1 is universal
+
+L2 dimensions (8):
+  1. accurate       — meaning preserved
+  2. fluent         — natural in target language
+  3. correct_lang   — in the target language (not source leakage)
+  4. no_leaks       — no prompt artifacts
+  5. terminology    — domain terms correctly handled
+  6. style          — appropriate register (formal/informal/technical)
+  7. completeness   — no content dropped or added
+  8. formatting     — codes, numbers, units preserved
+
+L1 was a binary pass/fail. L2 returns per-dimension scores (0/1) so the
+caller can decide which dimensions matter for a given job.
+
+DESIGN CONSTRAINTS (same as L1):
+  - 100% API-based. No local models, no GPU.
+  - Async, with a hard timeout.
+  - Defensive: never raises.
+  - Output is structured JSON.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+import time
+from dataclasses import dataclass, field, asdict
+from typing import List, Optional, Tuple, Dict, Any
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+# ---------- Result dataclasses ----------
+
+@dataclass
+class L2DimensionVerdict:
+    """8-dimension verdict for a single chunk."""
+    accurate: bool = False
+    fluent: bool = False
+    correct_lang: bool = False
+    no_leaks: bool = False
+    terminology: bool = False
+    style: bool = False
+    completeness: bool = False
+    formatting: bool = False
+    reason: str = ""
+
+    @property
+    def passed_count(self) -> int:
+        return sum([
+            self.accurate, self.fluent, self.correct_lang, self.no_leaks,
+            self.terminology, self.style, self.completeness, self.formatting,
+        ])
+
+    @property
+    def total(self) -> int:
+        return 8
+
+    @property
+    def score(self) -> float:
+        return self.passed_count / self.total
+
+    @property
+    def passed(self) -> bool:
+        # L2 is conservative: any single fail = chunk fails
+        return self.passed_count == self.total
+
+    def to_log_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class L2Result:
+    """Aggregate result of an L2 check on a sample of chunks."""
+    verdict: str                  # "pass", "fail", "skip"
+    chunks_evaluated: int = 0
+    chunks_passed: int = 0
+    chunks_failed: int = 0
+    failure_rate: float = 0.0
+    average_score: float = 0.0    # mean of per-chunk scores (0.0 to 1.0)
+    dimension_pass_rates: Dict[str, float] = field(default_factory=dict)
+    samples: List[dict] = field(default_factory=list)
+    model_used: str = ""
+    elapsed_ms: float = 0.0
+    cost_estimate_usd: float = 0.0
+    error: str = ""
+
+    def to_log_dict(self) -> dict:
+        return asdict(self)
+
+
+# ---------- Prompt template (8 dimensions) ----------
+
+L2_JUDGE_SYSTEM_PROMPT = """You are an expert translation quality evaluator using MQM-inspired criteria.
+
+For each (SOURCE, TRANSLATION) pair, check these 8 criteria (yes/no for each):
+
+1. ACCURATE     — Does the translation preserve the meaning of the source?
+2. FLUENT       — Is the translation natural and grammatical in {target_lang_name}?
+3. CORRECT_LANG — Is the translation actually in {target_lang_name} (ISO: {target_lang})?
+4. NO_LEAKS     — Is the translation free of prompt artifacts, source-language text, or meta-commentary?
+5. TERMINOLOGY  — Are domain-specific terms (technical, legal, medical, etc.) correctly translated?
+6. STYLE        — Is the register/tone appropriate (formal/informal/technical matching the source)?
+7. COMPLETENESS — Is all content present, with nothing added or dropped?
+8. FORMATTING   — Are codes, numbers, dates, and units preserved exactly?
+
+A translation FAILS if ANY criterion is "no". The "reason" must be in English and ≤ 20 words.
+
+Respond with a JSON array, one object per pair, in the same order. NO other text, NO markdown fences:
+[
+  {{"accurate": "yes"|"no", "fluent": "yes"|"no", "correct_lang": "yes"|"no", "no_leaks": "yes"|"no", "terminology": "yes"|"no", "style": "yes"|"no", "completeness": "yes"|"no", "formatting": "yes"|"no", "reason": "short justification"}}
+]
+"""
+
+
+# ---------- LLM client ----------
+
+class L2ProJudge:
+    """
+    Calls a STRONG LLM via the OpenAI-compatible API to judge translation
+    quality across 8 dimensions. Pro-tier only.
+
+    Default model: gpt-4o (strongest general judge we can afford at scale).
+    For sub-$0.01/job cost, we limit to 15 samples per job and 8 dimensions.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "https://api.openai.com/v1",
+        model: str = "gpt-4o",
+        timeout_seconds: float = 20.0,
+        max_retries: int = 1,
+    ):
+        if not api_key:
+            raise ValueError("api_key is required for L2ProJudge")
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        self._model = model
+        self._timeout = timeout_seconds
+        self._max_retries = max_retries
+        self._client = None
+
+    def _get_client(self):
+        if self._client is None:
+            try:
+                import openai
+                self._client = openai.AsyncOpenAI(
+                    api_key=self._api_key,
+                    base_url=self._base_url,
+                    timeout=self._timeout,
+                )
+            except Exception as e:
+                logger.warning("l2_judge_client_init_failed", error=str(e))
+                return None
+        return self._client
+
+    async def judge_batch(
+        self,
+        pairs: List[Tuple[str, str]],
+        target_lang: str,
+        target_lang_name: str = "",
+    ) -> L2Result:
+        """
+        Judge a batch of (source, translation) pairs across 8 dimensions.
+
+        Returns an L2Result with verdict="skip" on any internal error.
+        Never raises.
+        """
+        start = time.time()
+        empty = L2Result(verdict="skip", error="not_run")
+
+        if not pairs:
+            return L2Result(verdict="skip", error="empty pairs",
+                            elapsed_ms=round((time.time() - start) * 1000, 2))
+
+        client = self._get_client()
+        if client is None:
+            return L2Result(verdict="skip", error="client unavailable",
+                            elapsed_ms=round((time.time() - start) * 1000, 2))
+
+        user_lines = [f"Target language: {target_lang} ({target_lang_name})\n"]
+        for i, (src, trans) in enumerate(pairs, 1):
+            user_lines.append(f"\n--- Pair {i} ---")
+            user_lines.append(f"SOURCE: {src}")
+            user_lines.append(f"TRANSLATION: {trans}")
+        user_msg = "\n".join(user_lines)
+
+        system_prompt = L2_JUDGE_SYSTEM_PROMPT.format(
+            target_lang=target_lang,
+            target_lang_name=target_lang_name or target_lang,
+        )
+
+        try:
+            response = await asyncio.wait_for(
+                self._call_with_retries(client, system_prompt, user_msg),
+                timeout=self._timeout + 5.0,
+            )
+        except asyncio.TimeoutError:
+            elapsed_ms = round((time.time() - start) * 1000, 2)
+            logger.warning("l2_judge_timeout",
+                           timeout_s=self._timeout, elapsed_ms=elapsed_ms)
+            return L2Result(verdict="skip", error="timeout", elapsed_ms=elapsed_ms)
+        except Exception as e:
+            elapsed_ms = round((time.time() - start) * 1000, 2)
+            logger.warning("l2_judge_error",
+                           error=str(e)[:200], elapsed_ms=elapsed_ms)
+            return L2Result(verdict="skip", error=str(e)[:200],
+                            elapsed_ms=elapsed_ms)
+
+        verdicts = self._parse_response(response, len(pairs))
+
+        if not verdicts:
+            elapsed_ms = round((time.time() - start) * 1000, 2)
+            return L2Result(verdict="skip", error="parse_failed",
+                            elapsed_ms=elapsed_ms)
+
+        # Aggregate
+        passed = sum(1 for v in verdicts if v.passed)
+        failed = len(verdicts) - passed
+        failure_rate = failed / len(verdicts) if verdicts else 0.0
+        average_score = sum(v.score for v in verdicts) / len(verdicts)
+
+        # Per-dimension pass rates
+        dimensions = [
+            "accurate", "fluent", "correct_lang", "no_leaks",
+            "terminology", "style", "completeness", "formatting",
+        ]
+        dim_pass_rates = {}
+        for dim in dimensions:
+            count = sum(1 for v in verdicts if getattr(v, dim))
+            dim_pass_rates[dim] = round(count / len(verdicts), 3) if verdicts else 0.0
+
+        # L2 verdict: strict — any chunk fail = overall fail
+        verdict = "pass" if failed == 0 else "fail"
+
+        elapsed_ms = round((time.time() - start) * 1000, 2)
+        cost_estimate = self._estimate_cost(len(pairs))
+
+        return L2Result(
+            verdict=verdict,
+            chunks_evaluated=len(verdicts),
+            chunks_passed=passed,
+            chunks_failed=failed,
+            failure_rate=round(failure_rate, 3),
+            average_score=round(average_score, 3),
+            dimension_pass_rates=dim_pass_rates,
+            samples=[v.to_log_dict() for v in verdicts],
+            model_used=self._model,
+            elapsed_ms=elapsed_ms,
+            cost_estimate_usd=cost_estimate,
+        )
+
+    async def _call_with_retries(self, client, system_prompt: str, user_msg: str):
+        """Call the LLM with retry on transient errors."""
+        last_exc = None
+        for attempt in range(self._max_retries + 1):
+            try:
+                response = await client.chat.completions.create(
+                    model=self._model,
+                    messages=[
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_msg},
+                    ],
+                    temperature=0.0,
+                    max_tokens=1200,          # larger than L1 (more dimensions)
+                    response_format={"type": "json_object"},
+                )
+                return response
+            except Exception as e:
+                last_exc = e
+                if attempt < self._max_retries:
+                    await asyncio.sleep(0.8)
+        raise last_exc
+
+    def _parse_response(self, response, expected_count: int) -> List[L2DimensionVerdict]:
+        """Parse the LLM response into a list of 8-dimension verdicts."""
+        try:
+            content = response.choices[0].message.content or ""
+        except (AttributeError, IndexError) as e:
+            logger.warning("l2_judge_bad_response", error=str(e))
+            return []
+
+        content = content.strip()
+        if content.startswith("```"):
+            content = re.sub(r"^```(?:json)?\s*\n?", "", content)
+            content = re.sub(r"\n?```\s*$", "", content)
+
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError as e:
+            logger.warning("l2_judge_json_parse_error",
+                           error=str(e), content_preview=content[:200])
+            return []
+
+        if isinstance(data, dict):
+            items = None
+            for key in ("verdicts", "results", "translations", "data"):
+                if key in data and isinstance(data[key], list):
+                    items = data[key]
+                    break
+            if items is None:
+                for v in data.values():
+                    if isinstance(v, list):
+                        items = v
+                        break
+            if items is None:
+                logger.warning("l2_judge_no_list_in_response")
+                return []
+        elif isinstance(data, list):
+            items = data
+        else:
+            logger.warning("l2_judge_unexpected_response_type",
+                           type_=type(data).__name__)
+            return []
+
+        verdicts: List[L2DimensionVerdict] = []
+        for item in items:
+            try:
+                v = L2DimensionVerdict(
+                    accurate=str(item.get("accurate", "")).lower() == "yes",
+                    fluent=str(item.get("fluent", "")).lower() == "yes",
+                    correct_lang=str(item.get("correct_lang", "")).lower() == "yes",
+                    no_leaks=str(item.get("no_leaks", "")).lower() == "yes",
+                    terminology=str(item.get("terminology", "")).lower() == "yes",
+                    style=str(item.get("style", "")).lower() == "yes",
+                    completeness=str(item.get("completeness", "")).lower() == "yes",
+                    formatting=str(item.get("formatting", "")).lower() == "yes",
+                    reason=str(item.get("reason", ""))[:300],
+                )
+                verdicts.append(v)
+            except Exception as e:
+                logger.warning("l2_judge_item_parse_error",
+                               error=str(e), item=str(item)[:200])
+
+        return verdicts
+
+    def _estimate_cost(self, num_pairs: int) -> float:
+        """Rough USD cost estimate for the call."""
+        # L2 has more dimensions = longer output
+        input_tokens = 250 + (num_pairs * 280)
+        output_tokens = num_pairs * 110
+        model_lower = self._model.lower()
+        # IMPORTANT: check 'mini' BEFORE full 'gpt-4o' because
+        # 'gpt-4o-mini' contains 'gpt-4o'.
+        if "gpt-4o-mini" in model_lower:
+            input_cost = input_tokens / 1_000_000 * 0.15
+            output_cost = output_tokens / 1_000_000 * 0.60
+        elif "gpt-4o" in model_lower:
+            input_cost = input_tokens / 1_000_000 * 2.50
+            output_cost = output_tokens / 1_000_000 * 10.00
+        elif "claude" in model_lower:
+            input_cost = input_tokens / 1_000_000 * 3.00
+            output_cost = output_tokens / 1_000_000 * 15.00
+        else:
+            # Generic conservative
+            input_cost = input_tokens / 1_000_000 * 1.00
+            output_cost = output_tokens / 1_000_000 * 3.00
+        return round(input_cost + output_cost, 6)
+
+
+# ---------- Convenience factory ----------
+
+def make_l2_judge_from_env() -> Optional[L2ProJudge]:
+    """
+    Build an L2ProJudge from environment variables. Returns None if
+    no API key is configured.
+
+    Reads:
+      - L2_JUDGE_API_KEY       (required)
+      - L2_JUDGE_BASE_URL      (default: OpenAI)
+      - L2_JUDGE_MODEL         (default: gpt-4o)
+      - L2_JUDGE_TIMEOUT       (default: 20.0)
+    """
+    import os
+    api_key = os.getenv("L2_JUDGE_API_KEY", "").strip()
+    if not api_key:
+        return None
+    return L2ProJudge(
+        api_key=api_key,
+        base_url=os.getenv("L2_JUDGE_BASE_URL", "https://api.openai.com/v1"),
+        model=os.getenv("L2_JUDGE_MODEL", "gpt-4o"),
+        timeout_seconds=float(os.getenv("L2_JUDGE_TIMEOUT", "20.0")),
+    )

services/quality/pipeline.py

@@ -23,6 +23,7 @@ from core.logging import get_logger
 from .script_detector import evaluate_document, DocumentQualityResult
 from .sampler import sample_chunks_for_l1
 from .llm_judge import L1Result, LLMJudge
+from .l2_judge import L2Result, L2ProJudge
 
 logger = get_logger(__name__)
 
@@ -271,3 +272,155 @@ def make_judge_from_env_safe() -> Optional[LLMJudge]:
     except Exception as e:
         logger.warning("l1_judge_init_failed", error=str(e)[:200])
         return None
+
+
+# ---------- L2 (Pro tier) ----------
+
+async def run_l2_check(
+    source_chunks: List[str],
+    translated_chunks: List[str],
+    target_lang: Optional[str],
+    l0_failed_indices: Optional[Set[int]] = None,
+    job_id: Optional[str] = None,
+    file_extension: Optional[str] = None,
+    max_samples: int = 15,
+    min_chunks: int = 20,
+    judge: Optional[L2ProJudge] = None,
+    log_only: bool = True,
+) -> L2Result:
+    """
+    Run the L2 Pro premium judge (8 dimensions, gpt-4o default).
+
+    Args:
+        source_chunks: Original texts.
+        translated_chunks: Translated texts.
+        target_lang: Target language code (e.g. "fr", "en").
+        l0_failed_indices: Indices that L0 flagged as bad — skipped.
+        job_id: For logging.
+        file_extension: For logging.
+        max_samples: How many chunks to send to the LLM.
+        min_chunks: Skip the check if document has fewer chunks.
+        judge: An L2ProJudge instance. If None, created from env vars.
+        log_only: If True, never propagate the verdict (observation mode).
+                  If False, the caller can decide what to do with the verdict.
+
+    Returns an L2Result. verdict="skip" on any internal error.
+    Never raises — defensive wrapper.
+    """
+    skip = L2Result(verdict="skip", error="not_run")
+
+    if l0_failed_indices is None:
+        l0_failed_indices = set()
+
+    # Sample (reuse the L1 sampler — it's just chunk selection, model-agnostic)
+    sample = sample_chunks_for_l1(
+        source_chunks, translated_chunks, l0_failed_indices,
+        max_samples=max_samples, min_chunks=min_chunks,
+    )
+    if not sample:
+        logger.info(
+            "quality_l2_check_skipped",
+            job_id=job_id,
+            reason="insufficient_chunks_or_all_flagged",
+            chunk_count=len(source_chunks),
+        )
+        _record_l2_metric(verdict="skip", model="none")
+        return skip
+
+    # Get the judge
+    if judge is None:
+        judge = make_l2_judge_from_env_safe()
+
+    if judge is None:
+        logger.info(
+            "quality_l2_check_skipped",
+            job_id=job_id,
+            reason="no_l2_judge_configured",
+        )
+        _record_l2_metric(verdict="skip", model="none")
+        return skip
+
+    # Get the language name for the prompt
+    target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto")
+
+    # Call the LLM
+    try:
+        result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name)
+    except Exception as e:
+        logger.warning(
+            "quality_l2_check_failed",
+            job_id=job_id,
+            error=str(e)[:200],
+            error_type=type(e).__name__,
+        )
+        _record_l2_metric(verdict="error", model="unknown")
+        return L2Result(verdict="skip", error=str(e)[:200])
+
+    # Log (always) — caller decides what to do
+    logger.info(
+        "quality_l2_check",
+        job_id=job_id,
+        file_extension=file_extension,
+        target_lang=target_lang,
+        verdict=result.verdict,
+        chunks_evaluated=result.chunks_evaluated,
+        chunks_passed=result.chunks_passed,
+        chunks_failed=result.chunks_failed,
+        failure_rate=result.failure_rate,
+        average_score=result.average_score,
+        dimension_pass_rates=result.dimension_pass_rates,
+        model=result.model_used,
+        elapsed_ms=result.elapsed_ms,
+        cost_estimate_usd=result.cost_estimate_usd,
+        log_only=log_only,
+    )
+
+    # Record Prometheus metric
+    duration_s = None
+    if result.elapsed_ms is not None:
+        try:
+            duration_s = float(result.elapsed_ms) / 1000.0
+        except Exception:
+            duration_s = None
+    _record_l2_metric(
+        verdict=result.verdict or "skip",
+        model=result.model_used or "unknown",
+        duration_seconds=duration_s,
+        cost_usd=result.cost_estimate_usd,
+    )
+    return result
+
+
+def _record_l2_metric(
+    verdict: str,
+    model: str = "unknown",
+    duration_seconds: float = None,
+    cost_usd: float = None,
+) -> None:
+    """Best-effort Prometheus metric emission for L2.
+
+    Never raises.
+    """
+    try:
+        from middleware.metrics import record_l2_verdict
+        record_l2_verdict(
+            verdict=verdict,
+            model=model,
+            duration_seconds=duration_seconds,
+            cost_usd=cost_usd,
+        )
+    except Exception:
+        pass
+
+
+def make_l2_judge_from_env_safe() -> Optional[L2ProJudge]:
+    """Read env vars and build an L2 judge, or return None if not configured.
+
+    Defensive wrapper — a misconfigured L2 environment NEVER breaks a job.
+    """
+    try:
+        from .l2_judge import make_l2_judge_from_env
+        return make_l2_judge_from_env()
+    except Exception as e:
+        logger.warning("l2_judge_init_failed", error=str(e)[:200])
+        return None