feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s

Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions

View File

@@ -198,9 +198,11 @@ def build_full_prompt(
source_lang: str = "fr",
target_lang: str = "en",
glossary_target_lang: str = "multi",
formality: Optional[str] = None,
) -> str:
"""
Build the complete prompt combining custom prompt and glossary.
Build the complete prompt combining custom prompt, glossary, formality
and regional variant directives.
Args:
custom_prompt: Optional custom system prompt from user
@@ -208,6 +210,8 @@ def build_full_prompt(
source_lang: ISO code of the source language
target_lang: ISO code of the target language
glossary_target_lang: ISO code of the glossary's target language configuration
formality: Optional tone override — "formal" or "informal". Only
meaningful for LLM engines (ignored by classic engines).
Returns:
Combined prompt string
@@ -224,4 +228,30 @@ def build_full_prompt(
if glossary_prompt:
parts.append(glossary_prompt)
if formality in ("formal", "informal"):
if formality == "formal":
parts.append(
"TONE: Use a formal, professional register throughout "
"(formal address (vous/Sie) where the language distinguishes; "
"no slang, no contractions where avoidable)."
)
else:
parts.append(
"TONE: Use an informal, natural register throughout "
"(tu-style address where the language distinguishes; "
"contractions welcome)."
)
# Regional variant: when the target code carries a region (pt-BR,
# fr-CA, zh-CN...), make the expected variety explicit — LLMs default
# to the dominant variant otherwise (pt-PT, fr-FR...).
if target_lang and "-" in target_lang and target_lang != "auto":
from core.languages import language_name
name = language_name(target_lang)
if name and name != target_lang:
parts.append(
f"REGIONAL VARIANT: write specifically in {name}."
)
return "\n\n".join(parts) if parts else ""

216
services/mistral_ocr.py Normal file
View File

@@ -0,0 +1,216 @@
"""
Mistral OCR client — text extraction for scanned PDFs.
Image-only PDFs have no extractable text layer: PyMuPDF sees empty pages
and the layout-preserving pipeline would output an empty document. This
client calls the Mistral OCR API to recover the text.
API reference (2026): POST https://api.mistral.ai/v1/ocr with Bearer auth,
body {"model", "document": {"type": "document_url", "document_url":
"data:application/pdf;base64,..."}, "pages": [0, 1, ...]}. The response
contains {"pages": [{"index", "markdown", "dimensions"}, ...]}.
The document is sent in chunks of PAGES_PER_REQUEST pages: it bounds the
request payload and lets us report progress page by page.
"""
import base64
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
import requests
from core.logging import get_logger
logger = get_logger(__name__)
MISTRAL_OCR_URL = "https://api.mistral.ai/v1/ocr"
DEFAULT_MODEL = "mistral-ocr-latest"
PAGES_PER_REQUEST = 8
MISTRAL_INVALID_KEY = "MISTRAL_INVALID_KEY"
MISTRAL_QUOTA_EXCEEDED = "MISTRAL_QUOTA_EXCEEDED"
MISTRAL_TIMEOUT = "MISTRAL_TIMEOUT"
MISTRAL_SERVICE_ERROR = "MISTRAL_SERVICE_ERROR"
class MistralOCRError(Exception):
"""Raised when the Mistral OCR API cannot extract the PDF text."""
def __init__(
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
):
self.code = code
self.message = message
self.details = details or {}
super().__init__(self.message)
class MistralOCRClient:
"""Thin synchronous client around the Mistral OCR endpoint."""
def __init__(
self,
api_key: str,
model: str = DEFAULT_MODEL,
timeout: int = 180,
max_retries: int = 2,
retry_delay: float = 2.0,
):
self._api_key = (api_key or "").strip()
self._model = model or DEFAULT_MODEL
self._timeout = timeout
self._max_retries = max_retries
self._retry_delay = retry_delay
def is_available(self) -> bool:
"""True when an API key is configured."""
return bool(self._api_key)
def _post_ocr(self, data_uri: str, pages: List[int]) -> List[Dict[str, Any]]:
"""POST one OCR request for the given 0-based page list, with retries."""
payload = {
"model": self._model,
"document": {"type": "document_url", "document_url": data_uri},
"pages": pages,
}
headers = {
"Authorization": f"Bearer {self._api_key}",
"Content-Type": "application/json",
}
last_error: Optional[Exception] = None
for attempt in range(self._max_retries + 1):
try:
response = requests.post(
MISTRAL_OCR_URL,
json=payload,
headers=headers,
timeout=self._timeout,
)
if response.status_code == 401:
raise MistralOCRError(
MISTRAL_INVALID_KEY,
"Clé API Mistral invalide (MISTRAL_API_KEY).",
{"status_code": 401},
)
if response.status_code in (402, 429):
raise MistralOCRError(
MISTRAL_QUOTA_EXCEEDED,
"Quota Mistral OCR épuisé ou limite de débit atteinte.",
{"status_code": response.status_code},
)
if response.status_code >= 500:
raise MistralOCRError(
MISTRAL_SERVICE_ERROR,
f"Service Mistral OCR indisponible (HTTP {response.status_code}).",
{"status_code": response.status_code},
)
if response.status_code != 200:
raise MistralOCRError(
MISTRAL_SERVICE_ERROR,
f"Erreur Mistral OCR (HTTP {response.status_code}): {response.text[:200]}",
{"status_code": response.status_code},
)
pages_out = response.json().get("pages", [])
if not pages_out:
raise MistralOCRError(
MISTRAL_SERVICE_ERROR,
"Réponse Mistral OCR vide.",
)
return pages_out
except MistralOCRError as e:
if e.code in (MISTRAL_INVALID_KEY, MISTRAL_QUOTA_EXCEEDED):
raise # not transient
last_error = e
except requests.exceptions.Timeout as e:
last_error = MistralOCRError(
MISTRAL_TIMEOUT,
f"Délai d'attente Mistral OCR dépassé ({self._timeout}s).",
)
except requests.exceptions.RequestException as e:
last_error = MistralOCRError(
MISTRAL_SERVICE_ERROR,
f"Mistral OCR injoignable: {str(e)[:150]}",
)
if attempt < self._max_retries:
delay = self._retry_delay * (2**attempt)
logger.info(
"mistral_ocr_retry",
attempt=attempt + 1,
delay_s=round(delay, 2),
error=last_error.code if last_error else "unknown",
)
time.sleep(delay)
raise last_error or MistralOCRError(
MISTRAL_SERVICE_ERROR, "Erreur Mistral OCR inconnue."
)
def extract_pdf_text(
self,
pdf_path: Path,
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
) -> List[str]:
"""OCR a PDF file and return one text (markdown) string per page.
Pages are processed in chunks of ``PAGES_PER_REQUEST``; the returned
list is ordered by page index, empty strings for pages OCR returned
nothing for.
"""
import fitz
pdf_path = Path(pdf_path)
data_b64 = base64.b64encode(pdf_path.read_bytes()).decode("ascii")
data_uri = f"data:application/pdf;base64,{data_b64}"
with fitz.open(str(pdf_path)) as doc:
total_pages = len(doc)
if total_pages == 0:
raise MistralOCRError(MISTRAL_SERVICE_ERROR, "PDF vide (0 page).")
chunks = [
list(range(start, min(start + PAGES_PER_REQUEST, total_pages)))
for start in range(0, total_pages, PAGES_PER_REQUEST)
]
page_texts: List[str] = [""] * total_pages
done_chunks = 0
for chunk in chunks:
pages_out = self._post_ocr(data_uri, chunk)
for page in pages_out:
idx = int(page.get("index", -1))
if 0 <= idx < total_pages:
page_texts[idx] = page.get("markdown", "") or ""
done_chunks += 1
logger.info(
"mistral_ocr_chunk_done",
pages_done=min(done_chunks * PAGES_PER_REQUEST, total_pages),
total_pages=total_pages,
)
if progress_callback and chunks:
pct = int(5 + 20 * done_chunks / len(chunks))
progress_callback(
{
"current": done_chunks,
"total": len(chunks),
"phase": f"OCR (Mistral) {min(done_chunks * PAGES_PER_REQUEST, total_pages)}/{total_pages}",
"paragraph": done_chunks,
"total_paragraphs": len(chunks),
"progress_override": pct,
}
)
extracted = sum(1 for t in page_texts if t.strip())
logger.info(
"mistral_ocr_extracted",
pages_with_text=extracted,
total_pages=total_pages,
)
return page_texts

View File

@@ -96,38 +96,47 @@ class ProvidersConfig:
DEEPSEEK_MAX_RETRIES: int = int(os.getenv("DEEPSEEK_MAX_RETRIES", "3"))
DEEPSEEK_RETRY_DELAY: float = float(os.getenv("DEEPSEEK_RETRY_DELAY", "1.0"))
# Minimax (direct API - m2.7, MiniMax-M1)
# Minimax (public OpenAI-compatible API - https://api.minimax.io)
MINIMAX_ENABLED: bool = os.getenv("MINIMAX_ENABLED", "false").lower() == "true"
MINIMAX_API_KEY: str = os.getenv("MINIMAX_API_KEY", "")
MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M1")
MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.chat/v1")
MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M3")
MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.io/v1")
MINIMAX_GROUP_ID: str = os.getenv("MINIMAX_GROUP_ID", "")
MINIMAX_TIMEOUT: int = int(os.getenv("MINIMAX_TIMEOUT", "60"))
MINIMAX_MAX_RETRIES: int = int(os.getenv("MINIMAX_MAX_RETRIES", "3"))
MINIMAX_RETRY_DELAY: float = float(os.getenv("MINIMAX_RETRY_DELAY", "1.0"))
# Fallback chain configuration
# General fallback chain (backward compatibility)
#
# IMPORTANT: the registry-based fallback (translate_with_fallback) only
# ever sees providers that _auto_register_providers() registers, i.e.
# google, deepl, openai, deepseek and minimax. The OpenAI-compatible
# shims (openrouter, openrouter_premium, zai) and google_cloud are wired
# directly in routes/translate_routes.py and are intentionally NOT part of
# the registry fallback chain — listing them here would make
# translate_with_fallback silently skip them with a "provider not
# registered" log on every call. Override via env if you know what you
# are doing.
FALLBACK_CHAIN: List[str] = [
name.strip()
for name in os.getenv(
"PROVIDER_FALLBACK_CHAIN", "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
"PROVIDER_FALLBACK_CHAIN", "google,deepl,openai,deepseek,minimax"
).split(",")
if name.strip()
]
# Mode-specific fallback chains
# Classic mode: Google Translate -> Google Cloud -> DeepL
# Classic mode: Google Translate -> DeepL
FALLBACK_CHAIN_CLASSIC: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,google_cloud,deepl").split(",")
for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,deepl").split(",")
if name.strip()
]
# LLM mode: cloud providers in order of cost/quality (no Ollama by default)
# LLM mode: cloud providers in order of cost/quality (registry-registered only)
FALLBACK_CHAIN_LLM: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_LLM", "openrouter,openrouter_premium,openai,deepseek,zai").split(",")
for name in os.getenv("FALLBACK_CHAIN_LLM", "openai,deepseek,minimax").split(",")
if name.strip()
]

View File

@@ -39,16 +39,10 @@ Rules:
def _get_language_name(code: str) -> str:
language_names = {
"en": "English", "fr": "French", "es": "Spanish", "de": "German",
"it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
"zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
"hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
"th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
"ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
}
return language_names.get(code.split("-")[0].lower(), code)
"""Convert language code to full name (all supported languages)."""
from core.languages import language_name
return language_name(code)
class DeepSeekProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -161,9 +155,19 @@ class DeepSeekTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
_base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
# Base translation instructions always present; the custom prompt
# (glossary/tone/context) is appended, never a replacement.
if custom_prompt and custom_prompt.strip():
system_prompt = (
_base_prompt
+ "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ custom_prompt.strip()
)
else:
system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):

View File

@@ -17,7 +17,15 @@ import time
from core.logging import get_logger
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
# Detect structlog rather than hardcoding True, so the stdlib-logging fallback
# branches below stay reachable if structlog is ever absent.
try:
import structlog # noqa: F401
_HAS_STRUCTLOG = True
except ImportError: # pragma: no cover - structlog is a hard project dep
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):

View File

@@ -1,7 +1,9 @@
"""
Minimax Provider - Cloud LLM translation via Minimax API (m2.7).
Minimax Provider - Cloud LLM translation via the Minimax public API.
Minimax uses an OpenAI-compatible Chat Completions API.
Minimax exposes an OpenAI-compatible Chat Completions API at
``https://api.minimax.io/v1/chat/completions`` (default model ``MiniMax-M3``).
Note: ``api.minimax.chat`` is NOT a reachable public host.
"""
import threading
@@ -39,16 +41,10 @@ Rules:
def _get_language_name(code: str) -> str:
language_names = {
"en": "English", "fr": "French", "es": "Spanish", "de": "German",
"it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
"zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
"hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
"th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
"ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
}
return language_names.get(code.split("-")[0].lower(), code)
"""Convert language code to full name (all supported languages)."""
from core.languages import language_name
return language_name(code)
class MinimaxProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -62,17 +58,19 @@ class MinimaxTranslationProvider(TranslationProvider):
"""
Minimax translation provider using OpenAI-compatible API.
Default model: MiniMax-M1 (latest). Also supports m2.7 via env config.
Default model: MiniMax-M3 (latest public OpenAI-compatible model).
The public endpoint is https://api.minimax.io/v1 (NOT api.minimax.chat,
which is not a reachable host on the public API).
"""
def __init__(
self,
api_key: str,
model: str = "MiniMax-M1",
model: str = "MiniMax-M3",
timeout: int = 60,
max_retries: int = 3,
retry_delay: float = 1.0,
base_url: str = "https://api.minimax.chat/v1",
base_url: str = "https://api.minimax.io/v1",
group_id: str = "",
):
if not api_key or not api_key.strip():
@@ -144,13 +142,24 @@ class MinimaxTranslationProvider(TranslationProvider):
def get_name(self) -> str:
return self._provider_name
def is_available(self) -> bool:
def _probe_available(self) -> tuple[bool, int]:
"""Probe the Minimax API. Returns (available, status_code).
Minimax does not document a public ``GET /models`` endpoint, so a 404/405
on that path does NOT mean the provider is down — it only means the path
is absent. We treat any non-401 response as "available": the host is
reachable and the API key was not rejected. Only 401 (and network
errors) mark the provider unavailable.
"""
try:
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
return response.status_code == 200
return response.status_code != 401, response.status_code
except Exception:
return False
return False, 0
def is_available(self) -> bool:
return self._probe_available()[0]
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
text = request.text
@@ -163,9 +172,19 @@ class MinimaxTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
_base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
# Base translation instructions always present; the custom prompt
# (glossary/tone/context) is appended, never a replacement.
if custom_prompt and custom_prompt.strip():
system_prompt = (
_base_prompt
+ "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ custom_prompt.strip()
)
else:
system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):
@@ -199,20 +218,19 @@ class MinimaxTranslationProvider(TranslationProvider):
def health_check(self) -> ProviderHealthStatus:
start_time = time.time()
try:
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
latency_ms = (time.time() - start_time) * 1000
available, status_code = self._probe_available()
latency_ms = (time.time() - start_time) * 1000
if available:
return ProviderHealthStatus(
name=self._provider_name, available=response.status_code == 200,
name=self._provider_name, available=True,
latency_ms=round(latency_ms, 2), last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
except Exception as e:
return ProviderHealthStatus(
name=self._provider_name, available=False,
latency_ms=round((time.time() - start_time) * 1000, 2),
error=str(e)[:100], last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
return ProviderHealthStatus(
name=self._provider_name, available=False,
latency_ms=round(latency_ms, 2),
error=f"probe failed (status={status_code})"[:100],
last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
_provider_instance: Optional[MinimaxTranslationProvider] = None

View File

@@ -22,7 +22,16 @@ from typing import Any, Dict, List, Optional
from core.logging import get_logger
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
# structlog is the project's logger backend (see core/logging.py), but detect
# it rather than hardcoding True so the stdlib-logging fallback branches below
# are reachable if structlog is ever absent.
try:
import structlog # noqa: F401
_HAS_STRUCTLOG = True
except ImportError: # pragma: no cover - structlog is a hard project dep
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
@@ -111,56 +120,26 @@ Rules:
def _build_system_prompt(
source_lang: str, target_lang: str, custom_prompt: Optional[str] = None
) -> str:
"""Build system prompt for translation."""
if custom_prompt:
return custom_prompt
return DEFAULT_TRANSLATION_PROMPT.format(
"""Build system prompt for translation.
The base translation instructions are ALWAYS present — a custom prompt
(glossary, tone, context) is appended as additional directives, never a
replacement. Previously a glossary-only custom prompt produced a system
prompt with no translation instruction at all.
"""
base = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang, target_lang=target_lang
)
if custom_prompt and custom_prompt.strip():
return f"{base}\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n{custom_prompt.strip()}"
return base
def _get_language_name(code: str) -> str:
"""Convert language code to full name for better LLM understanding."""
language_names = {
"en": "English",
"fr": "French",
"es": "Spanish",
"de": "German",
"it": "Italian",
"pt": "Portuguese",
"nl": "Dutch",
"ru": "Russian",
"zh": "Chinese",
"ja": "Japanese",
"ko": "Korean",
"ar": "Arabic",
"hi": "Hindi",
"tr": "Turkish",
"pl": "Polish",
"vi": "Vietnamese",
"th": "Thai",
"id": "Indonesian",
"ms": "Malay",
"uk": "Ukrainian",
"cs": "Czech",
"sv": "Swedish",
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"el": "Greek",
"he": "Hebrew",
"ro": "Romanian",
"hu": "Hungarian",
"bg": "Bulgarian",
"sk": "Slovak",
"hr": "Croatian",
"sl": "Slovenian",
"lt": "Lithuanian",
"lv": "Latvian",
"et": "Estonian",
}
base_code = code.split("-")[0].lower()
return language_names.get(base_code, code)
from core.languages import language_name
return language_name(code)
class OpenAITranslationProvider(TranslationProvider):
@@ -529,21 +508,120 @@ class OpenAITranslationProvider(TranslationProvider):
error_code=OPENAI_SERVICE_ERROR,
)
def _make_batch_api_request(
self, texts: List[str], system_prompt: str
) -> Optional[List[str]]:
"""Translate a whole chunk in ONE request via a numbered JSON list.
The user message is a JSON array; the model must answer with a JSON
array of the same length. Returns None when the answer cannot be
parsed confidently — callers then fall back to per-item calls
(correctness over latency).
"""
import json as _json
numbered = _json.dumps(
[{"id": i, "text": t} for i, t in enumerate(texts)],
ensure_ascii=False,
)
batch_system = (
system_prompt
+ "\n\nBATCH MODE: the user message is a JSON array of items with "
"unique ids. Answer with ONLY a JSON array of objects "
'[{"id": <same id>, "translation": "<translated text>"}], same '
"length and same ids, in the same order. Translate every item; "
"keep ids unchanged; no comments, no markdown fence."
)
try:
content, _usage = self._make_api_request(numbered, batch_system)
raw = content.strip()
# Strip an optional markdown fence
if raw.startswith("```"):
raw = raw.strip("`")
if raw.lower().startswith("json"):
raw = raw[4:]
raw = raw.strip()
parsed = _json.loads(raw)
if not isinstance(parsed, list) or len(parsed) != len(texts):
return None
out: List[str] = [""] * len(texts)
for item in parsed:
if not isinstance(item, dict):
return None
idx = item.get("id")
translation = item.get("translation")
if not isinstance(idx, int) or not 0 <= idx < len(texts):
return None
if not isinstance(translation, str) or not translation.strip():
return None
out[idx] = translation.strip()
return out
except OpenAIProviderError:
raise
except Exception:
return None
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
Chunks arrive from the translators as ~15 texts. When every request
shares the same language pair and metadata, they are sent in ONE
call (numbered JSON list — ~15× fewer requests, better contextual
consistency across neighbouring segments). Any parse/API doubt falls
back to the per-item path so a batch failure never corrupts output.
"""
if not requests:
return []
same_pair = len({(r.source_language, r.target_language) for r in requests}) == 1
same_meta = len(
{tuple(sorted((r.metadata or {}).items())) for r in requests}
) == 1
if same_pair and same_meta and len(requests) > 1:
try:
source_lang_name = _get_language_name(
requests[0].source_language or "auto"
) or "the source language (auto-detect)"
target_lang_name = _get_language_name(requests[0].target_language)
custom_prompt = None
if requests[0].metadata:
custom_prompt = requests[0].metadata.get("custom_prompt")
system_prompt = _build_system_prompt(
source_lang_name, target_lang_name, custom_prompt
)
texts = [r.text for r in requests]
translations = self._make_batch_api_request(texts, system_prompt)
if translations is not None:
_log_info(
"openai_batch_translation_success",
items=len(requests),
model=self._model,
)
return [
TranslationResponse(
translated_text=t,
provider_name=self._provider_name,
from_cache=False,
)
for t in translations
]
_log_warning(
"openai_batch_translation_fallback",
reason="unparseable_response",
items=len(requests),
)
except Exception as e:
_log_warning(
"openai_batch_translation_fallback",
reason=type(e).__name__,
items=len(requests),
)
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:

View File

@@ -0,0 +1,119 @@
"""
Post-translation QA report (no external API — pure heuristics).
Answers three user-facing questions about a finished job:
- Are numbers preserved? (digit-token multiset source vs translation)
- Did everything actually get translated? (untranslated-ratio heuristic)
- A 0-100 confidence score combining both.
Never blocks a job: every failure degrades to "skipped".
"""
import re
from pathlib import Path
from typing import Dict, List, Optional
from core.logging import get_logger
logger = get_logger(__name__)
# Tokens that are never counted as "content words" for the untranslated ratio
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
_WORD_RE = re.compile(r"[\w']+", re.UNICODE)
_NUM_RE = re.compile(r"\d+(?:[.,]\d+)*", re.UNICODE)
# Latin-script vs non-Latin word detection for language confusion heuristics
_LATIN_RE = re.compile(r"[a-zA-Z]")
def _extract_text_pairs(source_path: Path, output_path: Path, file_extension: str):
"""Extract (source_text, translated_text) full-document strings.
Reuses the quality layer's file extractor so every format is read the
same way the L0 check reads it. Applied to the INPUT file the same
extractor yields the SOURCE text (the "translated" field simply holds
whatever text lives in the file).
"""
from services.quality.file_extractor import extract_sample
src_chunks = extract_sample(Path(source_path), file_extension, max_samples=10_000)
out_chunks = extract_sample(Path(output_path), file_extension, max_samples=10_000)
src = "\n".join(c["translated"] for c in src_chunks)
out = "\n".join(c["translated"] for c in out_chunks)
return src, out
def _number_multiset(text: str) -> List[str]:
"""Digit tokens with the decimal separator normalized (12,50 == 12.50).
French/English differ on ',' vs '.'; a real translation keeps the value.
"""
return sorted(n.replace(",", ".") for n in _NUM_RE.findall(text))
def _number_fidelity(source: str, translated: str) -> Optional[dict]:
"""Compare digit tokens: how many source numbers survived (order-insensitive)."""
src_nums = _number_multiset(source)
if not src_nums:
return None
out_nums = _number_multiset(translated)
# multiset intersection
from collections import Counter
src_count = Counter(src_nums)
out_count = Counter(out_nums)
kept = sum((src_count & out_count).values())
return {
"source_numbers": len(src_nums),
"preserved": kept,
"fidelity": round(kept / len(src_nums), 3),
}
def _untranslated_ratio(source: str, translated: str) -> Optional[float]:
"""Heuristic: share of source content-words still present verbatim in
the output. ~0 for a real translation, ~1 when nothing was translated.
"""
src_words = [w.lower() for w in _WORD_RE.findall(source) if len(w) > 3]
if len(src_words) < 10:
return None
out_lower = translated.lower()
hits = sum(1 for w in set(src_words) if w in out_lower)
return round(hits / len(set(src_words)), 3)
def run_qa_report(
source_path: Path, output_path: Path, target_lang: str, file_extension: str
) -> Optional[Dict]:
"""Compute the QA report for a finished translation job.
Returns a dict with numbers fidelity, untranslated ratio and a
0-100 score, or None if the report could not be computed.
"""
try:
source, translated = _extract_text_pairs(
Path(source_path), Path(output_path), file_extension
)
except Exception as e:
logger.warning("qa_report_extract_failed", error=str(e))
return None
if not source.strip() or not translated.strip():
return None
numbers = _number_fidelity(source, translated)
untranslated = _untranslated_ratio(source, translated)
score = 100.0
if numbers:
score *= 0.5 + 0.5 * numbers["fidelity"]
if untranslated is not None and untranslated > 0:
score *= max(0.0, 1.0 - untranslated)
report = {
"score": int(round(score)),
"numbers": numbers,
"untranslated_ratio": untranslated,
}
logger.info("qa_report_computed", **{k: v for k, v in report.items() if v is not None})
return report

View File

@@ -23,23 +23,11 @@ from core.logging import get_logger
logger = get_logger(__name__)
# Map language codes to full names for LLM prompts (models understand "French" better than "fr")
_LLM_LANG_NAMES = {
"en": "English", "es": "Spanish", "de": "German", "fr": "French", "ja": "Japanese",
"pt": "Portuguese", "ru": "Russian", "it": "Italian", "zh": "Chinese", "zh-CN": "Chinese (Simplified)",
"zh-TW": "Chinese (Traditional)", "pl": "Polish", "nl": "Dutch", "tr": "Turkish", "ko": "Korean",
"ar": "Arabic", "fa": "Persian", "vi": "Vietnamese", "id": "Indonesian", "uk": "Ukrainian",
"sv": "Swedish", "cs": "Czech", "el": "Greek", "he": "Hebrew", "hi": "Hindi", "ro": "Romanian",
"da": "Danish", "fi": "Finnish", "no": "Norwegian", "hu": "Hungarian", "th": "Thai",
"sk": "Slovak", "bg": "Bulgarian", "hr": "Croatian", "ca": "Catalan", "ms": "Malay",
}
def _lang_name(code: str) -> str:
"""Return full language name for LLM prompts; fallback to code if unknown."""
if not code or code == "auto":
return ""
return _LLM_LANG_NAMES.get(code, _LLM_LANG_NAMES.get(code.split("-")[0], code))
from core.languages import language_name
return language_name(code)
# Global thread pool for parallel translations
@@ -1191,13 +1179,16 @@ class TranslationService:
if not self.translate_images:
return ""
# Ollama, OpenAI, and OpenRouter support image translation
if isinstance(self.provider, OllamaTranslationProvider):
return self.provider.translate_image(image_path, target_language)
elif isinstance(self.provider, OpenAITranslationProvider):
return self.provider.translate_image(image_path, target_language)
elif isinstance(self.provider, OpenRouterTranslationProvider):
return self.provider.translate_image(image_path, target_language)
# Duck-typing: any provider that exposes ``translate_image`` can be
# used, regardless of whether it is a new-style (services/providers/*)
# or legacy (services/translation_service) instance. The previous
# isinstance() checks only matched the legacy classes, so a new-style
# provider wired in by the route never reached this branch.
if hasattr(self.provider, "translate_image"):
try:
return self.provider.translate_image(image_path, target_language)
except Exception:
return ""
return ""

199
services/translation_tm.py Normal file
View File

@@ -0,0 +1,199 @@
"""
Translation Memory (TM) — persistent reuse layer on top of the existing
Redis-backed cache (services/translation_cache.py).
Scope is PER USER (privacy: a customer's translations are never served to
another customer) and per translation context (custom prompt / glossary /
formality are hashed into the key so a glossary change invalidates old
matches). When Redis is not configured the cache transparently falls back
to a process-local LRU — still useful within a worker's lifetime.
"""
import hashlib
from typing import Dict, List, Optional, Tuple
from core.logging import get_logger
logger = get_logger(__name__)
class TMScope:
"""Identity/context of a translation job for TM keying."""
__slots__ = ("user_id", "context_hash")
def __init__(self, user_id: Optional[str], context_hash: Optional[str]):
self.user_id = user_id
self.context_hash = context_hash
@classmethod
def from_prompt(cls, user_id: Optional[str], prompt: Optional[str]) -> "TMScope":
"""Build a scope; the prompt (glossary + tone + formality merged)
is hashed so any directive change produces different TM entries."""
context_hash = None
if prompt:
context_hash = hashlib.sha256(prompt.encode("utf-8")).hexdigest()[:16]
return cls(user_id=user_id, context_hash=context_hash)
def _cache():
from services.translation_cache import get_cache
return get_cache()
def lookup_tm(
texts: List[str],
target_language: str,
source_language: str,
provider_name: str,
scope: Optional[TMScope],
) -> Tuple[Dict[int, str], List[int]]:
"""Return ({index: cached_translation}, [indices not in TM]).
No-op (all misses) when the scope has no user_id — anonymous jobs are
never cached or served from another user's entries.
"""
if not scope or not scope.user_id:
return {}, list(range(len(texts)))
try:
cache = _cache()
except Exception as e:
logger.warning("tm_init_failed", error=str(e))
return {}, list(range(len(texts)))
hits: Dict[int, str] = {}
misses: List[int] = []
for i, text in enumerate(texts):
if not text or not text.strip():
misses.append(i)
continue
cached = cache.get(
text,
target_language,
source_language,
provider_name,
user_id=scope.user_id,
custom_prompt_hash=scope.context_hash,
)
if cached is not None and cached.strip():
hits[i] = cached
else:
misses.append(i)
if hits:
logger.info("tm_lookup_hits", hits=len(hits), misses=len(misses))
return hits, misses
def store_tm(
texts: List[str],
translations: List[str],
target_language: str,
source_language: str,
provider_name: str,
scope: Optional[TMScope],
) -> int:
"""Store fresh translations in the TM. Returns the number stored."""
if not scope or not scope.user_id:
return 0
try:
cache = _cache()
except Exception as e:
logger.warning("tm_init_failed", error=str(e))
return 0
stored = 0
for text, translation in zip(texts, translations):
if not text or not translation:
continue
# Never store identity entries — they would poison future lookups
# (an unchanged text is not a translation).
if translation.strip() == text.strip():
continue
cache.set(
text,
target_language,
source_language,
provider_name,
translation,
user_id=scope.user_id,
custom_prompt_hash=scope.context_hash,
)
stored += 1
if stored:
logger.info("tm_stored", entries=stored)
return stored
def tm_stats() -> Dict:
"""Backend stats for observability."""
try:
return _cache().stats()
except Exception:
return {}
def translate_with_tm(
texts: List[str],
target_language: str,
source_language: str,
provider_name: str,
scope: "TMScope | None",
translate_fn,
) -> List[str]:
"""Translate a batch with TM reuse: hits come from the cache, misses go
through ``translate_fn`` (which receives ONLY the missed texts, in
order) and fresh results are stored back. Falls back to a plain
``translate_fn(texts)`` call when the TM is unavailable.
"""
if not texts:
return []
tm_hits, miss_indices = lookup_tm(
texts, target_language, source_language, provider_name, scope
)
miss_texts = [texts[i] for i in miss_indices]
if not miss_texts:
return [tm_hits.get(i, texts[i]) for i in range(len(texts))]
try:
translated_misses = translate_fn(miss_texts)
except Exception:
if tm_hits:
# Provider failed but we have partial TM hits — keep them and
# leave the misses untouched rather than dropping everything.
logger.warning("tm_translate_fn_failed_partial_hits", hits=len(tm_hits))
translated_misses = None
else:
raise
if translated_misses is None:
return [
tm_hits.get(i, texts[i]) for i in range(len(texts))
]
store_tm(
miss_texts,
translated_misses,
target_language,
source_language,
provider_name,
scope,
)
merged: List[str] = []
miss_pos = 0
for i in range(len(texts)):
if i in tm_hits:
merged.append(tm_hits[i])
else:
merged.append(
translated_misses[miss_pos]
if miss_pos < len(translated_misses)
else texts[i]
)
miss_pos += 1
return merged