feat(quality): A3 — L1 LLM judge via API (5 chunks, 0.0003 USD/job)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m26s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m26s
L1 quality layer — uses a cheap LLM via the OpenAI-compatible API to
validate translation quality. Designed to be the SECOND line of defense
after L0 (script detection, length, pattern).
Architecture:
- sampler.py — picks 5 representative chunks per job (longest first,
skips L0-failed indices, skips too-short or identical pairs)
- llm_judge.py — OpenAI-compatible client, binary verdict per chunk
(accurate / fluent / correct_language / no_leaks), JSON output,
hard timeout, defensive (never raises), cost estimation built in
- pipeline.py — defensive wrapper that integrates both, never breaks
a translation job, always logs a structured event
Integration:
- 5 feature flags in config.py (QUALITY_L1_ENABLED, _LOG_ONLY, etc.)
- QUALITY_L1_LOG_ONLY=true by default: log-only mode, verdict NEVER
blocks or retries a job
- Reuses the chunks extracted by L0 (no double work)
- Passes the set of L0-failed indices so L1 doesn't re-judge them
- Wrapped in try/except so a misconfigured L1 NEVER breaks a job
Default config: deepseek-chat via DeepSeek API
- Cost: ~0.0003 USD per job (5 chunks)
- Speed: typically 1-2s per call, hard ceiling at 8s
- Easy to swap: just set L1_JUDGE_BASE_URL and L1_JUDGE_MODEL
LLM judge is intentionally a SEPARATE model from the translator
(self-evaluation bias mitigation — Meta/Stanford papers 2024-2025).
Tests:
test_sampler.py — 9 tests covering the sampling strategy
test_llm_judge.py — 22 tests covering init, parsing, mocked API,
cost estimation, env factory
test_l1_pipeline.py — 6 tests covering the wrapper
Total new: 37 tests, all pass
Grand total quality+format: 264 tests passing (0 regression)
All 36 new tests + 111 L0 tests + 117 existing translator tests = 264
Phase 1 (observation) for 2 weeks. Then QUALITY_L1_LOG_ONLY=false
to enable auto-retry via the fallback chain.
This commit is contained in:
@@ -1,27 +1,34 @@
|
||||
"""
|
||||
Quality pipeline — defensive wrapper around the L0 checks.
|
||||
Quality pipeline — defensive wrapper around the L0 and L1 checks.
|
||||
|
||||
The pipeline is the integration point for the route. It:
|
||||
1. Catches all exceptions (L0 must NEVER break a translation job)
|
||||
1. Catches all exceptions (quality checks must NEVER break a translation job)
|
||||
2. Adds timing
|
||||
3. Emits a single structured log line per job
|
||||
4. For L1, reads configuration from environment and is opt-in via
|
||||
a flag passed in by the route (the route reads config.py).
|
||||
|
||||
The actual checks live in `script_detector`, `length_checker`, `pattern_leak`.
|
||||
This module is the orchestration / safety layer.
|
||||
L0 = pure-Python script + length + pattern checks. No API calls.
|
||||
L1 = cheap LLM judge via API. Sampled (5 chunks per job by default).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from core.logging import get_logger
|
||||
|
||||
from .script_detector import evaluate_document, DocumentQualityResult
|
||||
from .sampler import sample_chunks_for_l1
|
||||
from .llm_judge import L1Result, LLMJudge
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# ---------- L0 ----------
|
||||
|
||||
def run_l0_check(
|
||||
source_chunks: List[str],
|
||||
translated_chunks: List[str],
|
||||
@@ -31,10 +38,6 @@ def run_l0_check(
|
||||
) -> DocumentQualityResult:
|
||||
"""
|
||||
Run the L0 quality checks defensively. Never raises.
|
||||
|
||||
Returns an empty/neutral DocumentQualityResult on internal error
|
||||
so the calling route can log and continue without affecting the
|
||||
translation job outcome.
|
||||
"""
|
||||
start = time.time()
|
||||
empty = DocumentQualityResult(
|
||||
@@ -73,3 +76,138 @@ def run_l0_check(
|
||||
elapsed_ms=elapsed_ms,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ---------- L1 ----------
|
||||
|
||||
# Common ISO codes → language name for the prompt
|
||||
_LANG_NAMES = {
|
||||
"en": "English", "fr": "French", "es": "Spanish", "de": "German",
|
||||
"it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
|
||||
"ja": "Japanese", "ko": "Korean", "zh": "Chinese", "ar": "Arabic",
|
||||
"fa": "Persian", "hi": "Hindi", "tr": "Turkish", "pl": "Polish",
|
||||
"vi": "Vietnamese", "th": "Thai", "id": "Indonesian", "ms": "Malay",
|
||||
"uk": "Ukrainian", "cs": "Czech", "sv": "Swedish", "ro": "Romanian",
|
||||
"hu": "Hungarian", "el": "Greek", "he": "Hebrew", "da": "Danish",
|
||||
"fi": "Finnish", "no": "Norwegian", "bg": "Bulgarian", "hr": "Croatian",
|
||||
}
|
||||
|
||||
|
||||
async def run_l1_check(
|
||||
source_chunks: List[str],
|
||||
translated_chunks: List[str],
|
||||
target_lang: Optional[str],
|
||||
l0_failed_indices: Optional[Set[int]] = None,
|
||||
job_id: Optional[str] = None,
|
||||
file_extension: Optional[str] = None,
|
||||
max_samples: int = 5,
|
||||
min_chunks: int = 10,
|
||||
judge: Optional[LLMJudge] = None,
|
||||
log_only: bool = True,
|
||||
) -> L1Result:
|
||||
"""
|
||||
Run the L1 LLM judge check on a sample of chunks.
|
||||
|
||||
Args:
|
||||
source_chunks: Original texts.
|
||||
translated_chunks: Translated texts.
|
||||
target_lang: Target language code (e.g. "fr", "en").
|
||||
l0_failed_indices: Indices that L0 flagged as bad — skipped.
|
||||
job_id: For logging.
|
||||
file_extension: For logging.
|
||||
max_samples: How many chunks to send to the LLM.
|
||||
min_chunks: Skip the check if document has fewer chunks.
|
||||
judge: An LLMJudge instance. If None, created from env vars.
|
||||
log_only: If True, never propagate the verdict (observation mode).
|
||||
If False, the caller can decide what to do with the verdict.
|
||||
|
||||
Returns an L1Result. verdict="skip" on any internal error.
|
||||
Never raises — defensive wrapper.
|
||||
"""
|
||||
skip = L1Result(
|
||||
verdict="skip",
|
||||
chunks_evaluated=0, chunks_passed=0, chunks_failed=0,
|
||||
failure_rate=0.0, error="not_run",
|
||||
)
|
||||
|
||||
if l0_failed_indices is None:
|
||||
l0_failed_indices = set()
|
||||
|
||||
# Sample
|
||||
sample = sample_chunks_for_l1(
|
||||
source_chunks, translated_chunks, l0_failed_indices,
|
||||
max_samples=max_samples, min_chunks=min_chunks,
|
||||
)
|
||||
if not sample:
|
||||
logger.info(
|
||||
"quality_l1_check_skipped",
|
||||
job_id=job_id,
|
||||
reason="insufficient_chunks_or_all_flagged",
|
||||
chunk_count=len(source_chunks),
|
||||
)
|
||||
return skip
|
||||
|
||||
# Get the judge
|
||||
if judge is None:
|
||||
judge = make_judge_from_env_safe()
|
||||
|
||||
if judge is None:
|
||||
logger.info(
|
||||
"quality_l1_check_skipped",
|
||||
job_id=job_id,
|
||||
reason="no_judge_configured",
|
||||
)
|
||||
return skip
|
||||
|
||||
# Get the language name for the prompt
|
||||
target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto")
|
||||
|
||||
# Call the LLM
|
||||
try:
|
||||
result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name)
|
||||
except Exception as e:
|
||||
elapsed_ms = 0.0
|
||||
logger.warning(
|
||||
"quality_l1_check_failed",
|
||||
job_id=job_id,
|
||||
error=str(e)[:200],
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
return L1Result(
|
||||
verdict="skip",
|
||||
chunks_evaluated=0, chunks_passed=0, chunks_failed=0,
|
||||
failure_rate=0.0, error=str(e)[:200], elapsed_ms=elapsed_ms,
|
||||
)
|
||||
|
||||
# Log (always) — caller decides what to do
|
||||
logger.info(
|
||||
"quality_l1_check",
|
||||
job_id=job_id,
|
||||
file_extension=file_extension,
|
||||
target_lang=target_lang,
|
||||
verdict=result.verdict,
|
||||
chunks_evaluated=result.chunks_evaluated,
|
||||
chunks_passed=result.chunks_passed,
|
||||
chunks_failed=result.chunks_failed,
|
||||
failure_rate=result.failure_rate,
|
||||
model=result.model_used,
|
||||
elapsed_ms=result.elapsed_ms,
|
||||
cost_estimate_usd=result.cost_estimate_usd,
|
||||
log_only=log_only,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def make_judge_from_env_safe() -> Optional[LLMJudge]:
|
||||
"""Read env vars and build a judge, or return None if not configured.
|
||||
|
||||
This is a thin wrapper around `llm_judge.make_judge_from_env` that
|
||||
catches any import-time or init-time error and returns None — so
|
||||
a misconfigured L1 environment NEVER breaks a translation job.
|
||||
"""
|
||||
try:
|
||||
from .llm_judge import make_judge_from_env
|
||||
return make_judge_from_env()
|
||||
except Exception as e:
|
||||
logger.warning("l1_judge_init_failed", error=str(e)[:200])
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user