office_translator/services/quality/pipeline.py

"""
Quality pipeline — defensive wrapper around the L0 and L1 checks.

The pipeline is the integration point for the route. It:
  1. Catches all exceptions (quality checks must NEVER break a translation job)
  2. Adds timing
  3. Emits a single structured log line per job
  4. For L1, reads configuration from environment and is opt-in via
     a flag passed in by the route (the route reads config.py).

L0 = pure-Python script + length + pattern checks. No API calls.
L1 = cheap LLM judge via API. Sampled (5 chunks per job by default).
"""

from __future__ import annotations

import os
import time
from typing import List, Optional, Set

from core.logging import get_logger

from .script_detector import evaluate_document, DocumentQualityResult
from .sampler import sample_chunks_for_l1
from .llm_judge import L1Result, LLMJudge
from .l2_judge import L2Result, L2ProJudge

logger = get_logger(__name__)


# ---------- L0 ----------

def run_l0_check(
    source_chunks: List[str],
    translated_chunks: List[str],
    target_lang: Optional[str],
    job_id: Optional[str] = None,
    file_extension: Optional[str] = None,
) -> DocumentQualityResult:
    """
    Run the L0 quality checks defensively. Never raises.
    """
    start = time.time()
    empty = DocumentQualityResult(
        passed=True,
        score=0.0,
        chunk_count=0,
        failed_chunk_count=0,
        issues={"internal_error": 1},
    )

    # Best-effort import of metrics; never let it break the check.
    try:
        from middleware.metrics import record_l0_result, record_l0_error
    except Exception:
        record_l0_result = None
        record_l0_error = None

    try:
        result = evaluate_document(source_chunks, translated_chunks, target_lang)
    except Exception as e:
        elapsed_ms = round((time.time() - start) * 1000, 2)
        logger.warning(
            "quality_l0_check_failed",
            job_id=job_id,
            file_extension=file_extension,
            error=str(e)[:200],
            error_type=type(e).__name__,
            elapsed_ms=elapsed_ms,
        )
        if record_l0_error:
            try:
                record_l0_error(file_type=file_extension or "unknown")
            except Exception:
                pass
        return empty

    elapsed_ms = round((time.time() - start) * 1000, 2)
    logger.info(
        "quality_l0_check",
        job_id=job_id,
        file_extension=file_extension,
        target_lang=target_lang,
        chunk_count=result.chunk_count,
        failed_chunk_count=result.failed_chunk_count,
        score=result.score,
        passed=result.passed,
        issues=result.issues,
        elapsed_ms=elapsed_ms,
    )
    if record_l0_result:
        try:
            record_l0_result(
                passed=bool(result.passed),
                file_type=file_extension or "unknown",
            )
        except Exception:
            pass
    return result


# ---------- L1 ----------

# Common ISO codes → language name for the prompt
_LANG_NAMES = {
    "en": "English", "fr": "French", "es": "Spanish", "de": "German",
    "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
    "ja": "Japanese", "ko": "Korean", "zh": "Chinese", "ar": "Arabic",
    "fa": "Persian", "hi": "Hindi", "tr": "Turkish", "pl": "Polish",
    "vi": "Vietnamese", "th": "Thai", "id": "Indonesian", "ms": "Malay",
    "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish", "ro": "Romanian",
    "hu": "Hungarian", "el": "Greek", "he": "Hebrew", "da": "Danish",
    "fi": "Finnish", "no": "Norwegian", "bg": "Bulgarian", "hr": "Croatian",
}


async def run_l1_check(
    source_chunks: List[str],
    translated_chunks: List[str],
    target_lang: Optional[str],
    l0_failed_indices: Optional[Set[int]] = None,
    job_id: Optional[str] = None,
    file_extension: Optional[str] = None,
    max_samples: int = 5,
    min_chunks: int = 10,
    judge: Optional[LLMJudge] = None,
    log_only: bool = True,
) -> L1Result:
    """
    Run the L1 LLM judge check on a sample of chunks.

    Args:
        source_chunks: Original texts.
        translated_chunks: Translated texts.
        target_lang: Target language code (e.g. "fr", "en").
        l0_failed_indices: Indices that L0 flagged as bad — skipped.
        job_id: For logging.
        file_extension: For logging.
        max_samples: How many chunks to send to the LLM.
        min_chunks: Skip the check if document has fewer chunks.
        judge: An LLMJudge instance. If None, created from env vars.
        log_only: If True, never propagate the verdict (observation mode).
                  If False, the caller can decide what to do with the verdict.

    Returns an L1Result. verdict="skip" on any internal error.
    Never raises — defensive wrapper.
    """
    skip = L1Result(
        verdict="skip",
        chunks_evaluated=0, chunks_passed=0, chunks_failed=0,
        failure_rate=0.0, error="not_run",
    )

    if l0_failed_indices is None:
        l0_failed_indices = set()

    # Sample
    sample = sample_chunks_for_l1(
        source_chunks, translated_chunks, l0_failed_indices,
        max_samples=max_samples, min_chunks=min_chunks,
    )
    if not sample:
        logger.info(
            "quality_l1_check_skipped",
            job_id=job_id,
            reason="insufficient_chunks_or_all_flagged",
            chunk_count=len(source_chunks),
        )
        _record_l1_metric(verdict="skip", model="none")
        return skip

    # Get the judge
    if judge is None:
        judge = make_judge_from_env_safe()

    if judge is None:
        logger.info(
            "quality_l1_check_skipped",
            job_id=job_id,
            reason="no_judge_configured",
        )
        _record_l1_metric(verdict="skip", model="none")
        return skip

    # Get the language name for the prompt
    target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto")

    # Call the LLM
    try:
        result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name)
    except Exception as e:
        elapsed_ms = 0.0
        logger.warning(
            "quality_l1_check_failed",
            job_id=job_id,
            error=str(e)[:200],
            error_type=type(e).__name__,
        )
        _record_l1_metric(verdict="error", model="unknown")
        return L1Result(
            verdict="skip",
            chunks_evaluated=0, chunks_passed=0, chunks_failed=0,
            failure_rate=0.0, error=str(e)[:200], elapsed_ms=elapsed_ms,
        )

    # Log (always) — caller decides what to do
    logger.info(
        "quality_l1_check",
        job_id=job_id,
        file_extension=file_extension,
        target_lang=target_lang,
        verdict=result.verdict,
        chunks_evaluated=result.chunks_evaluated,
        chunks_passed=result.chunks_passed,
        chunks_failed=result.chunks_failed,
        failure_rate=result.failure_rate,
        model=result.model_used,
        elapsed_ms=result.elapsed_ms,
        cost_estimate_usd=result.cost_estimate_usd,
        log_only=log_only,
    )

    # Record Prometheus metric (best-effort, never breaks the job)
    duration_s = None
    if result.elapsed_ms is not None:
        try:
            duration_s = float(result.elapsed_ms) / 1000.0
        except Exception:
            duration_s = None
    _record_l1_metric(
        verdict=result.verdict or "skip",
        model=result.model_used or "unknown",
        duration_seconds=duration_s,
        cost_usd=result.cost_estimate_usd,
    )
    return result


def _record_l1_metric(
    verdict: str,
    model: str = "unknown",
    duration_seconds: float = None,
    cost_usd: float = None,
) -> None:
    """Best-effort Prometheus metric emission for L1.

    Never raises — if the metrics module fails to import or the
    counter is unavailable, we silently skip.
    """
    try:
        from middleware.metrics import record_l1_verdict
        record_l1_verdict(
            verdict=verdict,
            model=model,
            duration_seconds=duration_seconds,
            cost_usd=cost_usd,
        )
    except Exception:
        pass


def make_judge_from_env_safe() -> Optional[LLMJudge]:
    """Read env vars and build a judge, or return None if not configured.

    This is a thin wrapper around `llm_judge.make_judge_from_env` that
    catches any import-time or init-time error and returns None — so
    a misconfigured L1 environment NEVER breaks a translation job.
    """
    try:
        from .llm_judge import make_judge_from_env
        return make_judge_from_env()
    except Exception as e:
        logger.warning("l1_judge_init_failed", error=str(e)[:200])
        return None


# ---------- L2 (Pro tier) ----------

async def run_l2_check(
    source_chunks: List[str],
    translated_chunks: List[str],
    target_lang: Optional[str],
    l0_failed_indices: Optional[Set[int]] = None,
    job_id: Optional[str] = None,
    file_extension: Optional[str] = None,
    max_samples: int = 15,
    min_chunks: int = 20,
    judge: Optional[L2ProJudge] = None,
    log_only: bool = True,
) -> L2Result:
    """
    Run the L2 Pro premium judge (8 dimensions, gpt-4o default).

    Args:
        source_chunks: Original texts.
        translated_chunks: Translated texts.
        target_lang: Target language code (e.g. "fr", "en").
        l0_failed_indices: Indices that L0 flagged as bad — skipped.
        job_id: For logging.
        file_extension: For logging.
        max_samples: How many chunks to send to the LLM.
        min_chunks: Skip the check if document has fewer chunks.
        judge: An L2ProJudge instance. If None, created from env vars.
        log_only: If True, never propagate the verdict (observation mode).
                  If False, the caller can decide what to do with the verdict.

    Returns an L2Result. verdict="skip" on any internal error.
    Never raises — defensive wrapper.
    """
    skip = L2Result(verdict="skip", error="not_run")

    if l0_failed_indices is None:
        l0_failed_indices = set()

    # Sample (reuse the L1 sampler — it's just chunk selection, model-agnostic)
    sample = sample_chunks_for_l1(
        source_chunks, translated_chunks, l0_failed_indices,
        max_samples=max_samples, min_chunks=min_chunks,
    )
    if not sample:
        logger.info(
            "quality_l2_check_skipped",
            job_id=job_id,
            reason="insufficient_chunks_or_all_flagged",
            chunk_count=len(source_chunks),
        )
        _record_l2_metric(verdict="skip", model="none")
        return skip

    # Get the judge
    if judge is None:
        judge = make_l2_judge_from_env_safe()

    if judge is None:
        logger.info(
            "quality_l2_check_skipped",
            job_id=job_id,
            reason="no_l2_judge_configured",
        )
        _record_l2_metric(verdict="skip", model="none")
        return skip

    # Get the language name for the prompt
    target_lang_name = _LANG_NAMES.get((target_lang or "").lower(), target_lang or "auto")

    # Call the LLM
    try:
        result = await judge.judge_batch(sample, target_lang or "auto", target_lang_name)
    except Exception as e:
        logger.warning(
            "quality_l2_check_failed",
            job_id=job_id,
            error=str(e)[:200],
            error_type=type(e).__name__,
        )
        _record_l2_metric(verdict="error", model="unknown")
        return L2Result(verdict="skip", error=str(e)[:200])

    # Log (always) — caller decides what to do
    logger.info(
        "quality_l2_check",
        job_id=job_id,
        file_extension=file_extension,
        target_lang=target_lang,
        verdict=result.verdict,
        chunks_evaluated=result.chunks_evaluated,
        chunks_passed=result.chunks_passed,
        chunks_failed=result.chunks_failed,
        failure_rate=result.failure_rate,
        average_score=result.average_score,
        dimension_pass_rates=result.dimension_pass_rates,
        model=result.model_used,
        elapsed_ms=result.elapsed_ms,
        cost_estimate_usd=result.cost_estimate_usd,
        log_only=log_only,
    )

    # Record Prometheus metric
    duration_s = None
    if result.elapsed_ms is not None:
        try:
            duration_s = float(result.elapsed_ms) / 1000.0
        except Exception:
            duration_s = None
    _record_l2_metric(
        verdict=result.verdict or "skip",
        model=result.model_used or "unknown",
        duration_seconds=duration_s,
        cost_usd=result.cost_estimate_usd,
    )
    return result


def _record_l2_metric(
    verdict: str,
    model: str = "unknown",
    duration_seconds: float = None,
    cost_usd: float = None,
) -> None:
    """Best-effort Prometheus metric emission for L2.

    Never raises.
    """
    try:
        from middleware.metrics import record_l2_verdict
        record_l2_verdict(
            verdict=verdict,
            model=model,
            duration_seconds=duration_seconds,
            cost_usd=cost_usd,
        )
    except Exception:
        pass


def make_l2_judge_from_env_safe() -> Optional[L2ProJudge]:
    """Read env vars and build an L2 judge, or return None if not configured.

    Defensive wrapper — a misconfigured L2 environment NEVER breaks a job.
    """
    try:
        from .l2_judge import make_l2_judge_from_env
        return make_l2_judge_from_env()
    except Exception as e:
        logger.warning("l2_judge_init_failed", error=str(e)[:200])
        return None