sepehr/office_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Details

Browse Source 
Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
sepehr
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
services/providers/config.py
View File

@@ -96,38 +96,47 @@ class ProvidersConfig:
DEEPSEEK_MAX_RETRIES: int = int(os.getenv("DEEPSEEK_MAX_RETRIES", "3"))
DEEPSEEK_RETRY_DELAY: float = float(os.getenv("DEEPSEEK_RETRY_DELAY", "1.0"))
# Minimax (direct API - m2.7, MiniMax-M1)
# Minimax (public OpenAI-compatible API - https://api.minimax.io)
MINIMAX_ENABLED: bool = os.getenv("MINIMAX_ENABLED", "false").lower() == "true"
MINIMAX_API_KEY: str = os.getenv("MINIMAX_API_KEY", "")
MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M1")
MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.chat/v1")
MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M3")
MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.io/v1")
MINIMAX_GROUP_ID: str = os.getenv("MINIMAX_GROUP_ID", "")
MINIMAX_TIMEOUT: int = int(os.getenv("MINIMAX_TIMEOUT", "60"))
MINIMAX_MAX_RETRIES: int = int(os.getenv("MINIMAX_MAX_RETRIES", "3"))
MINIMAX_RETRY_DELAY: float = float(os.getenv("MINIMAX_RETRY_DELAY", "1.0"))
# Fallback chain configuration
# General fallback chain (backward compatibility)
#
# IMPORTANT: the registry-based fallback (translate_with_fallback) only
# ever sees providers that _auto_register_providers() registers, i.e.
# google, deepl, openai, deepseek and minimax. The OpenAI-compatible
# shims (openrouter, openrouter_premium, zai) and google_cloud are wired
# directly in routes/translate_routes.py and are intentionally NOT part of
# the registry fallback chain — listing them here would make
# translate_with_fallback silently skip them with a "provider not
# registered" log on every call. Override via env if you know what you
# are doing.
FALLBACK_CHAIN: List[str] = [
name.strip()
for name in os.getenv(
"PROVIDER_FALLBACK_CHAIN", "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
"PROVIDER_FALLBACK_CHAIN", "google,deepl,openai,deepseek,minimax"
).split(",")
if name.strip()
]
# Mode-specific fallback chains
# Classic mode: Google Translate -> Google Cloud -> DeepL
# Classic mode: Google Translate -> DeepL
FALLBACK_CHAIN_CLASSIC: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,google_cloud,deepl").split(",")
for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,deepl").split(",")
if name.strip()
]
# LLM mode: cloud providers in order of cost/quality (no Ollama by default)
# LLM mode: cloud providers in order of cost/quality (registry-registered only)
FALLBACK_CHAIN_LLM: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_LLM", "openrouter,openrouter_premium,openai,deepseek,zai").split(",")
for name in os.getenv("FALLBACK_CHAIN_LLM", "openai,deepseek,minimax").split(",")
if name.strip()
]

24
services/providers/deepseek_provider.py
View File

@@ -39,16 +39,10 @@ Rules:
def _get_language_name(code: str) -> str:
language_names = {
"en": "English", "fr": "French", "es": "Spanish", "de": "German",
"it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
"zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
"hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
"th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
"ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
}
return language_names.get(code.split("-")[0].lower(), code)
"""Convert language code to full name (all supported languages)."""
from core.languages import language_name
return language_name(code)
class DeepSeekProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -161,9 +155,19 @@ class DeepSeekTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
_base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
# Base translation instructions always present; the custom prompt
# (glossary/tone/context) is appended, never a replacement.
if custom_prompt and custom_prompt.strip():
system_prompt = (
_base_prompt
+ "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ custom_prompt.strip()
)
else:
system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):

10
services/providers/fallback.py
View File

@@ -17,7 +17,15 @@ import time
from core.logging import get_logger
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
# Detect structlog rather than hardcoding True, so the stdlib-logging fallback
# branches below stay reachable if structlog is ever absent.
try:
import structlog # noqa: F401
_HAS_STRUCTLOG = True
except ImportError: # pragma: no cover - structlog is a hard project dep
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):

76
services/providers/minimax_provider.py
View File

@@ -1,7 +1,9 @@
"""
Minimax Provider - Cloud LLM translation via Minimax API (m2.7).
Minimax Provider - Cloud LLM translation via the Minimax public API.
Minimax uses an OpenAI-compatible Chat Completions API.
Minimax exposes an OpenAI-compatible Chat Completions API at
``https://api.minimax.io/v1/chat/completions`` (default model ``MiniMax-M3``).
Note: ``api.minimax.chat`` is NOT a reachable public host.
"""
import threading
@@ -39,16 +41,10 @@ Rules:
def _get_language_name(code: str) -> str:
language_names = {
"en": "English", "fr": "French", "es": "Spanish", "de": "German",
"it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
"zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
"hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
"th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
"ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
}
return language_names.get(code.split("-")[0].lower(), code)
"""Convert language code to full name (all supported languages)."""
from core.languages import language_name
return language_name(code)
class MinimaxProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -62,17 +58,19 @@ class MinimaxTranslationProvider(TranslationProvider):
"""
Minimax translation provider using OpenAI-compatible API.
Default model: MiniMax-M1 (latest). Also supports m2.7 via env config.
Default model: MiniMax-M3 (latest public OpenAI-compatible model).
The public endpoint is https://api.minimax.io/v1 (NOT api.minimax.chat,
which is not a reachable host on the public API).
"""
def __init__(
self,
api_key: str,
model: str = "MiniMax-M1",
model: str = "MiniMax-M3",
timeout: int = 60,
max_retries: int = 3,
retry_delay: float = 1.0,
base_url: str = "https://api.minimax.chat/v1",
base_url: str = "https://api.minimax.io/v1",
group_id: str = "",
):
if not api_key or not api_key.strip():
@@ -144,13 +142,24 @@ class MinimaxTranslationProvider(TranslationProvider):
def get_name(self) -> str:
return self._provider_name
def is_available(self) -> bool:
def _probe_available(self) -> tuple[bool, int]:
"""Probe the Minimax API. Returns (available, status_code).
Minimax does not document a public ``GET /models`` endpoint, so a 404/405
on that path does NOT mean the provider is down — it only means the path
is absent. We treat any non-401 response as "available": the host is
reachable and the API key was not rejected. Only 401 (and network
errors) mark the provider unavailable.
"""
try:
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
return response.status_code == 200
return response.status_code != 401, response.status_code
except Exception:
return False
return False, 0
def is_available(self) -> bool:
return self._probe_available()[0]
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
text = request.text
@@ -163,9 +172,19 @@ class MinimaxTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
_base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
# Base translation instructions always present; the custom prompt
# (glossary/tone/context) is appended, never a replacement.
if custom_prompt and custom_prompt.strip():
system_prompt = (
_base_prompt
+ "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ custom_prompt.strip()
)
else:
system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):
@@ -199,20 +218,19 @@ class MinimaxTranslationProvider(TranslationProvider):
def health_check(self) -> ProviderHealthStatus:
start_time = time.time()
try:
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
latency_ms = (time.time() - start_time) * 1000
available, status_code = self._probe_available()
latency_ms = (time.time() - start_time) * 1000
if available:
return ProviderHealthStatus(
name=self._provider_name, available=response.status_code == 200,
name=self._provider_name, available=True,
latency_ms=round(latency_ms, 2), last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
except Exception as e:
return ProviderHealthStatus(
name=self._provider_name, available=False,
latency_ms=round((time.time() - start_time) * 1000, 2),
error=str(e)[:100], last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
return ProviderHealthStatus(
name=self._provider_name, available=False,
latency_ms=round(latency_ms, 2),
error=f"probe failed (status={status_code})"[:100],
last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
_provider_instance: Optional[MinimaxTranslationProvider] = None

178
services/providers/openai_provider.py
View File

@@ -22,7 +22,16 @@ from typing import Any, Dict, List, Optional
from core.logging import get_logger
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
# structlog is the project's logger backend (see core/logging.py), but detect
# it rather than hardcoding True so the stdlib-logging fallback branches below
# are reachable if structlog is ever absent.
try:
import structlog # noqa: F401
_HAS_STRUCTLOG = True
except ImportError: # pragma: no cover - structlog is a hard project dep
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
@@ -111,56 +120,26 @@ Rules:
def _build_system_prompt(
source_lang: str, target_lang: str, custom_prompt: Optional[str] = None
) -> str:
"""Build system prompt for translation."""
if custom_prompt:
return custom_prompt
return DEFAULT_TRANSLATION_PROMPT.format(
"""Build system prompt for translation.
The base translation instructions are ALWAYS present — a custom prompt
(glossary, tone, context) is appended as additional directives, never a
replacement. Previously a glossary-only custom prompt produced a system
prompt with no translation instruction at all.
"""
base = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang, target_lang=target_lang
)
if custom_prompt and custom_prompt.strip():
return f"{base}\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n{custom_prompt.strip()}"
return base
def _get_language_name(code: str) -> str:
"""Convert language code to full name for better LLM understanding."""
language_names = {
"en": "English",
"fr": "French",
"es": "Spanish",
"de": "German",
"it": "Italian",
"pt": "Portuguese",
"nl": "Dutch",
"ru": "Russian",
"zh": "Chinese",
"ja": "Japanese",
"ko": "Korean",
"ar": "Arabic",
"hi": "Hindi",
"tr": "Turkish",
"pl": "Polish",
"vi": "Vietnamese",
"th": "Thai",
"id": "Indonesian",
"ms": "Malay",
"uk": "Ukrainian",
"cs": "Czech",
"sv": "Swedish",
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"el": "Greek",
"he": "Hebrew",
"ro": "Romanian",
"hu": "Hungarian",
"bg": "Bulgarian",
"sk": "Slovak",
"hr": "Croatian",
"sl": "Slovenian",
"lt": "Lithuanian",
"lv": "Latvian",
"et": "Estonian",
}
base_code = code.split("-")[0].lower()
return language_names.get(base_code, code)
from core.languages import language_name
return language_name(code)
class OpenAITranslationProvider(TranslationProvider):
@@ -529,21 +508,120 @@ class OpenAITranslationProvider(TranslationProvider):
error_code=OPENAI_SERVICE_ERROR,
)
def _make_batch_api_request(
self, texts: List[str], system_prompt: str
) -> Optional[List[str]]:
"""Translate a whole chunk in ONE request via a numbered JSON list.
The user message is a JSON array; the model must answer with a JSON
array of the same length. Returns None when the answer cannot be
parsed confidently — callers then fall back to per-item calls
(correctness over latency).
"""
import json as _json
numbered = _json.dumps(
[{"id": i, "text": t} for i, t in enumerate(texts)],
ensure_ascii=False,
)
batch_system = (
system_prompt
+ "\n\nBATCH MODE: the user message is a JSON array of items with "
"unique ids. Answer with ONLY a JSON array of objects "
'[{"id": <same id>, "translation": "<translated text>"}], same '
"length and same ids, in the same order. Translate every item; "
"keep ids unchanged; no comments, no markdown fence."
)
try:
content, _usage = self._make_api_request(numbered, batch_system)
raw = content.strip()
# Strip an optional markdown fence
if raw.startswith("```"):
raw = raw.strip("`")
if raw.lower().startswith("json"):
raw = raw[4:]
raw = raw.strip()
parsed = _json.loads(raw)
if not isinstance(parsed, list) or len(parsed) != len(texts):
return None
out: List[str] = [""] * len(texts)
for item in parsed:
if not isinstance(item, dict):
return None
idx = item.get("id")
translation = item.get("translation")
if not isinstance(idx, int) or not 0 <= idx < len(texts):
return None
if not isinstance(translation, str) or not translation.strip():
return None
out[idx] = translation.strip()
return out
except OpenAIProviderError:
raise
except Exception:
return None
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
Chunks arrive from the translators as ~15 texts. When every request
shares the same language pair and metadata, they are sent in ONE
call (numbered JSON list — ~15× fewer requests, better contextual
consistency across neighbouring segments). Any parse/API doubt falls
back to the per-item path so a batch failure never corrupts output.
"""
if not requests:
return []
same_pair = len({(r.source_language, r.target_language) for r in requests}) == 1
same_meta = len(
{tuple(sorted((r.metadata or {}).items())) for r in requests}
) == 1
if same_pair and same_meta and len(requests) > 1:
try:
source_lang_name = _get_language_name(
requests[0].source_language or "auto"
) or "the source language (auto-detect)"
target_lang_name = _get_language_name(requests[0].target_language)
custom_prompt = None
if requests[0].metadata:
custom_prompt = requests[0].metadata.get("custom_prompt")
system_prompt = _build_system_prompt(
source_lang_name, target_lang_name, custom_prompt
)
texts = [r.text for r in requests]
translations = self._make_batch_api_request(texts, system_prompt)
if translations is not None:
_log_info(
"openai_batch_translation_success",
items=len(requests),
model=self._model,
)
return [
TranslationResponse(
translated_text=t,
provider_name=self._provider_name,
from_cache=False,
)
for t in translations
]
_log_warning(
"openai_batch_translation_fallback",
reason="unparseable_response",
items=len(requests),
)
except Exception as e:
_log_warning(
"openai_batch_translation_fallback",
reason=type(e).__name__,
items=len(requests),
)
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus: