office_translator/services/quality/length_checker.py

"""
Length sanity check for the L0 quality layer.

A translation that's 10× longer or 10× shorter than the source is almost
certainly a hallucination or a truncation. We flag these as warnings
(not failures) so the caller can decide what to do.

Thresholds are tunable via env vars if needed, but the defaults work
well for prose documents. Tables and bullet lists will naturally have
shorter translations, so we keep the lower bound loose.

Special cases:
  * If the source is mostly digits/punctuation, the translation can also
    be short (e.g. "Price: 100$" → "100€") — skip the check.
  * If the source is empty/very short, skip the check entirely.
  * If the source contains an embedded URL or email, the translation may
    legitimately shrink — skip the check.
"""

from __future__ import annotations

import re
from typing import Dict

# Default thresholds — generous enough to handle tables / short strings.
RATIO_MAX = 3.5
RATIO_MIN = 0.15
# Hard lower bound: a translation shorter than this is very suspect,
# UNLESS the source is also very short.
ABSOLUTE_MIN_LENGTH = 2
# If source is short (under this many chars), skip the ratio check entirely
# — short strings are too noisy to be useful for length analysis.
MIN_SOURCE_LENGTH_FOR_RATIO = 20

# Pattern to detect text that is mostly digits / numbers / simple symbols.
# E.g. "Price: 100€", "+33 6 12 34 56 78", "192.168.1.1".
_MOSTLY_NUMERIC_RE = re.compile(r"^[\d\s\W]*$", re.UNICODE)
_NUMERIC_RATIO_THRESHOLD = 0.5  # 50% of letters are digits


def check(source_text: str, translated_text: str) -> Dict:
    """
    Returns a dict like:
        {
            "issue": None | "length_outlier" | "truncation_suspect",
            "ratio": float,
            "source_length": int,
            "translated_length": int,
        }
    Never raises.
    """
    if not source_text:
        return {
            "issue": None,
            "ratio": None,
            "source_length": 0,
            "translated_length": len(translated_text or ""),
        }

    src_len = len(source_text.strip())
    trans_len = len(translated_text.strip())

    # Empty translation — always suspect.
    if trans_len == 0:
        return {
            "issue": "truncation_suspect",
            "ratio": 0.0,
            "source_length": src_len,
            "translated_length": trans_len,
        }

    # If source is mostly digits/numbers, the translation can also be short
    # (e.g. "Price: 100" → "100"). Don't flag length in this case.
    if _is_mostly_numeric(source_text):
        return {
            "issue": None,
            "ratio": None,
            "source_length": src_len,
            "translated_length": trans_len,
            "note": "skipped: numeric source",
        }

    # If source is very short, skip the ratio check.
    if src_len < MIN_SOURCE_LENGTH_FOR_RATIO:
        return {
            "issue": None,
            "ratio": None,
            "source_length": src_len,
            "translated_length": trans_len,
        }

    ratio = trans_len / src_len

    if ratio > RATIO_MAX:
        return {
            "issue": "length_outlier",
            "ratio": round(ratio, 2),
            "source_length": src_len,
            "translated_length": trans_len,
        }
    if ratio < RATIO_MIN:
        return {
            "issue": "truncation_suspect",
            "ratio": round(ratio, 2),
            "source_length": src_len,
            "translated_length": trans_len,
        }

    return {
        "issue": None,
        "ratio": round(ratio, 2),
        "source_length": src_len,
        "translated_length": trans_len,
    }


def _is_mostly_numeric(text: str) -> bool:
    """True if at least 50% of non-whitespace characters are digits."""
    if not text:
        return False
    chars = [c for c in text if not c.isspace()]
    if not chars:
        return False
    digit_count = sum(1 for c in chars if c.isdigit())
    return (digit_count / len(chars)) >= _NUMERIC_RATIO_THRESHOLD