""" Pattern leak detection for the L0 quality layer. Detects common failure modes where the LLM translator: 1. Leaks parts of the system prompt (e.g. starts with "Translation:" or "Voici la traduction :") 2. Hallucinates by repeating the same word/phrase many times in a row 3. Returns a chain-of-thought / explanation instead of a translation These checks are pure regex / counting — no model, no network. """ from __future__ import annotations import re from typing import Dict, List # ---------- Prompt leak patterns ---------- # Phrases that strongly suggest the LLM leaked its prompt or a thought process # into the output. We check only at the START of the translation (after # stripping whitespace) to avoid false positives on legitimate text. LEAK_PREFIX_PATTERNS: List[re.Pattern] = [ re.compile(r"^(translation|translated text|here is the translation|here'?s the translation)\s*[::-]", re.IGNORECASE), re.compile(r"^(voici (la |ma )?traduction|traduction\s*[::-])\b", re.IGNORECASE), re.compile(r"^(原文|译|翻译|译为|以下是)\s*[::]?", re.UNICODE), re.compile(r"^(sure,?\s+here'?s?\s+(the\s+)?translation|of course,?\s+here)", re.IGNORECASE), re.compile(r"^(\*\*|__|\#)\s*translation", re.IGNORECASE), re.compile(r"^translated from\s+\w+\s+to\s+\w+\s*[::-]", re.IGNORECASE), ] # ---------- Repetition detection ---------- # A "word" for the repetition check — uses Unicode word boundaries so # it works on Chinese, Japanese, Korean, etc. _WORD_RE = re.compile(r"\S+", re.UNICODE) # Threshold: a word (or token) repeated 5+ times consecutively is almost # always a hallucination. We allow up to 4 to handle legitimate text # like "the the" (typo) without false positives. REPETITION_THRESHOLD = 5 # Same character repeated 20+ times in a row is also a hallucination # (catches cases like "xxxxxxxxxxx" or "=========="). CHAR_REPETITION_THRESHOLD = 20 def check(text: str) -> Dict: """ Returns a dict like: { "issue": None | "prompt_leak" | "repetition_hallucination", "matched_pattern": "..." | None, "repetition_count": int | None, } Never raises. """ if not text or not text.strip(): return {"issue": None, "matched_pattern": None, "repetition_count": None} stripped = text.lstrip() # 1. Prompt leak for pat in LEAK_PREFIX_PATTERNS: m = pat.match(stripped) if m: return { "issue": "prompt_leak", "matched_pattern": pat.pattern, "repetition_count": None, } # 2. Token-level repetition tokens = _WORD_RE.findall(stripped) rep_count = _max_consecutive_repetition(tokens) if rep_count >= REPETITION_THRESHOLD: return { "issue": "repetition_hallucination", "matched_pattern": None, "repetition_count": rep_count, } # 3. Character-level repetition (catches "xxxxxx" without spaces) char_rep = _max_consecutive_char_repetition(stripped) if char_rep >= CHAR_REPETITION_THRESHOLD: return { "issue": "repetition_hallucination", "matched_pattern": None, "repetition_count": char_rep, } return {"issue": None, "matched_pattern": None, "repetition_count": max(rep_count, char_rep) or None} def _max_consecutive_repetition(tokens: List[str]) -> int: """Return the maximum number of times the same token appears consecutively.""" if not tokens: return 0 # Normalize: lower-case + strip basic punctuation for comparison norm = [t.lower().strip(".,!?;:\"'`()[]{}") for t in tokens] max_run = 1 current_run = 1 for i in range(1, len(norm)): if norm[i] and norm[i] == norm[i - 1]: current_run += 1 if current_run > max_run: max_run = current_run else: current_run = 1 return max_run def _max_consecutive_char_repetition(text: str) -> int: """Return the maximum number of times the same character appears consecutively.""" if not text: return 0 max_run = 1 current_run = 1 for i in range(1, len(text)): if text[i] == text[i - 1] and not text[i].isspace(): current_run += 1 if current_run > max_run: max_run = current_run else: current_run = 1 return max_run