""" Quality pipeline — defensive wrapper around the L0 and L1 checks. The pipeline is the integration point for the route. It: 1. Catches all exceptions (quality checks must NEVER break a translation job) 2. Adds timing 3. Emits a single structured log line per job 4. For L1, reads configuration from environment and is opt-in via a flag passed in by the route (the route reads config.py). L0 = pure-Python script + length + pattern checks. No API calls. L1 = cheap LLM judge via API. Sampled (5 chunks per job by default). """ from __future__ import annotations import os import time from typing import List, Optional, Set from core.logging import get_logger from .script_detector import evaluate_document, DocumentQualityResult from .sampler import sample_chunks_for_l1 from .llm_judge import L1Result, LLMJudge from .l2_judge import L2Result, L2ProJudge logger = get_logger(__name__) # ---------- L0 ---------- def run_l0_check( source_chunks: List[str], translated_chunks: List[str], target_lang: Optional[str], job_id: Optional[str] = None, file_extension: Optional[str] = None, ) -> DocumentQualityResult: """ Run the L0 quality checks defensively. Never raises. """ start = time.time() empty = DocumentQualityResult( passed=True, score=0.0, chunk_count=0, failed_chunk_count=0, issues={"internal_error": 1}, ) # Best-effort import of metrics; never let it break the check. try: from middleware.metrics import record_l0_result, record_l0_error except Exception: record_l0_result = None record_l0_error = None try: result = evaluate_document(source_chunks, translated_chunks, target_lang) except Exception as e: elapsed_ms = round((time.time() - start) * 1000, 2) logger.warning( "quality_l0_check_failed", job_id=job_id, file_extension=file_extension, error=str(e)[:200], error_type=type(e).__name__, elapsed_ms=elapsed_ms, ) if record_l0_error: try: record_l0_error(file_type=file_extension or "unknown") except Exception: pass return empty elapsed_ms = round((time.time() - start) * 1000, 2) logger.info( "quality_l0_check", job_id=job_id, file_extension=file_extension, target_lang=target_lang, chunk_count=result.chunk_count, failed_chunk_count=result.failed_chunk_count, score=result.score, passed=result.passed, issues=result.issues, elapsed_ms=elapsed_ms, ) if record_l0_result: try: record_l0_result( passed=bool(result.passed), file_type=file_extension or "unknown", ) except Exception: pass return result # ---------- L1 ---------- # Common ISO codes → language name for the prompt _LANG_NAMES = { "en": "English", "fr": "French", "es": "Spanish", "de": "German", "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian", "ja": "Japanese", "ko": "Korean", "zh": "Chinese", "ar": "Arabic", "fa": "Persian", "hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese", "th": "Thai", "id": "Indonesian", "ms": "Malay", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew", "da": "Danish", "fi": "Finnish", "no": "Norwegian", "bg": "Bulgarian", "hr": "Croatian", } async def run_l1_check( source_chunks: List[str], translated_chunks: List[str], target_lang: Optional[str], l0_failed_indices: Optional[Set[int]] = None, job_id: Optional[str] = None, file_extension: Optional[str] = None, max_samples: int = 5, min_chunks: int = 10, judge: Optional[LLMJudge] = None, log_only: bool = True, ) -> L1Result: """ Run the L1 LLM judge check on a sample of chunks. Args: source_chunks: Original texts. translated_chunks: Translated texts. target_lang: Target language code (e.g. "fr", "en"). l0_failed_indices: Indices that L0 flagged as bad — skipped. job_id: For logging. file_extension: For logging. max_samples: How many chunks to send to the LLM. min_chunks: Skip the check if document has fewer chunks. judge: An LLMJudge instance. If None, created from env vars. log_only: If True, never propagate the verdict (observation mode). If False, the caller can decide what to do with the verdict. Returns an L1Result. verdict="skip" on any internal error. Never raises — defensive wrapper. """ skip = L1Result( verdict="skip", chunks_evaluated=0, chunks_passed=0, chunks_failed=0, failure_rate=0.0, error="not_run", ) if l0_failed_indices is None: l0_failed_indices = set() # Sample sample = sample_chunks_for_l1( source_chunks, translated_chunks, l0_failed_indices, max_samples=max_samples, min_chunks=min_chunks, ) if not sample: logger.info( "quality_l1_check_skipped", job_id=job_id, reason="insufficient_chunks_or_all_flagged", chunk_count=len(source_chunks), ) _record_l1_metric(verdict="skip", model="none") return skip # Get the judge if judge is None: judge = make_judge_from_env_safe() if judge is None: logger.info( "quality_l1_check_skipped", job_id=job_id, reason="no_judge_configured", ) _record_l1_metric(verdict="skip", model="none") return skip # Get the language name for the prompt target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto") # Call the LLM try: result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name) except Exception as e: elapsed_ms = 0.0 logger.warning( "quality_l1_check_failed", job_id=job_id, error=str(e)[:200], error_type=type(e).__name__, ) _record_l1_metric(verdict="error", model="unknown") return L1Result( verdict="skip", chunks_evaluated=0, chunks_passed=0, chunks_failed=0, failure_rate=0.0, error=str(e)[:200], elapsed_ms=elapsed_ms, ) # Log (always) — caller decides what to do logger.info( "quality_l1_check", job_id=job_id, file_extension=file_extension, target_lang=target_lang, verdict=result.verdict, chunks_evaluated=result.chunks_evaluated, chunks_passed=result.chunks_passed, chunks_failed=result.chunks_failed, failure_rate=result.failure_rate, model=result.model_used, elapsed_ms=result.elapsed_ms, cost_estimate_usd=result.cost_estimate_usd, log_only=log_only, ) # Record Prometheus metric (best-effort, never breaks the job) duration_s = None if result.elapsed_ms is not None: try: duration_s = float(result.elapsed_ms) / 1000.0 except Exception: duration_s = None _record_l1_metric( verdict=result.verdict or "skip", model=result.model_used or "unknown", duration_seconds=duration_s, cost_usd=result.cost_estimate_usd, ) return result def _record_l1_metric( verdict: str, model: str = "unknown", duration_seconds: float = None, cost_usd: float = None, ) -> None: """Best-effort Prometheus metric emission for L1. Never raises — if the metrics module fails to import or the counter is unavailable, we silently skip. """ try: from middleware.metrics import record_l1_verdict record_l1_verdict( verdict=verdict, model=model, duration_seconds=duration_seconds, cost_usd=cost_usd, ) except Exception: pass def make_judge_from_env_safe() -> Optional[LLMJudge]: """Read env vars and build a judge, or return None if not configured. This is a thin wrapper around `llm_judge.make_judge_from_env` that catches any import-time or init-time error and returns None — so a misconfigured L1 environment NEVER breaks a translation job. """ try: from .llm_judge import make_judge_from_env return make_judge_from_env() except Exception as e: logger.warning("l1_judge_init_failed", error=str(e)[:200]) return None # ---------- L2 (Pro tier) ---------- async def run_l2_check( source_chunks: List[str], translated_chunks: List[str], target_lang: Optional[str], l0_failed_indices: Optional[Set[int]] = None, job_id: Optional[str] = None, file_extension: Optional[str] = None, max_samples: int = 15, min_chunks: int = 20, judge: Optional[L2ProJudge] = None, log_only: bool = True, ) -> L2Result: """ Run the L2 Pro premium judge (8 dimensions, gpt-4o default). Args: source_chunks: Original texts. translated_chunks: Translated texts. target_lang: Target language code (e.g. "fr", "en"). l0_failed_indices: Indices that L0 flagged as bad — skipped. job_id: For logging. file_extension: For logging. max_samples: How many chunks to send to the LLM. min_chunks: Skip the check if document has fewer chunks. judge: An L2ProJudge instance. If None, created from env vars. log_only: If True, never propagate the verdict (observation mode). If False, the caller can decide what to do with the verdict. Returns an L2Result. verdict="skip" on any internal error. Never raises — defensive wrapper. """ skip = L2Result(verdict="skip", error="not_run") if l0_failed_indices is None: l0_failed_indices = set() # Sample (reuse the L1 sampler — it's just chunk selection, model-agnostic) sample = sample_chunks_for_l1( source_chunks, translated_chunks, l0_failed_indices, max_samples=max_samples, min_chunks=min_chunks, ) if not sample: logger.info( "quality_l2_check_skipped", job_id=job_id, reason="insufficient_chunks_or_all_flagged", chunk_count=len(source_chunks), ) _record_l2_metric(verdict="skip", model="none") return skip # Get the judge if judge is None: judge = make_l2_judge_from_env_safe() if judge is None: logger.info( "quality_l2_check_skipped", job_id=job_id, reason="no_l2_judge_configured", ) _record_l2_metric(verdict="skip", model="none") return skip # Get the language name for the prompt target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto") # Call the LLM try: result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name) except Exception as e: logger.warning( "quality_l2_check_failed", job_id=job_id, error=str(e)[:200], error_type=type(e).__name__, ) _record_l2_metric(verdict="error", model="unknown") return L2Result(verdict="skip", error=str(e)[:200]) # Log (always) — caller decides what to do logger.info( "quality_l2_check", job_id=job_id, file_extension=file_extension, target_lang=target_lang, verdict=result.verdict, chunks_evaluated=result.chunks_evaluated, chunks_passed=result.chunks_passed, chunks_failed=result.chunks_failed, failure_rate=result.failure_rate, average_score=result.average_score, dimension_pass_rates=result.dimension_pass_rates, model=result.model_used, elapsed_ms=result.elapsed_ms, cost_estimate_usd=result.cost_estimate_usd, log_only=log_only, ) # Record Prometheus metric duration_s = None if result.elapsed_ms is not None: try: duration_s = float(result.elapsed_ms) / 1000.0 except Exception: duration_s = None _record_l2_metric( verdict=result.verdict or "skip", model=result.model_used or "unknown", duration_seconds=duration_s, cost_usd=result.cost_estimate_usd, ) return result def _record_l2_metric( verdict: str, model: str = "unknown", duration_seconds: float = None, cost_usd: float = None, ) -> None: """Best-effort Prometheus metric emission for L2. Never raises. """ try: from middleware.metrics import record_l2_verdict record_l2_verdict( verdict=verdict, model=model, duration_seconds=duration_seconds, cost_usd=cost_usd, ) except Exception: pass def make_l2_judge_from_env_safe() -> Optional[L2ProJudge]: """Read env vars and build an L2 judge, or return None if not configured. Defensive wrapper — a misconfigured L2 environment NEVER breaks a job. """ try: from .l2_judge import make_l2_judge_from_env return make_l2_judge_from_env() except Exception as e: logger.warning("l2_judge_init_failed", error=str(e)[:200]) return None