- Drag & drop your .xlsx, .docx, or .pptx file here
+ Drag & drop your .xlsx, .docx, .pptx, or .pdf file here
or click to browse
diff --git a/office-translator-landing-page/components/waitlist-section.tsx b/office-translator-landing-page/components/waitlist-section.tsx
new file mode 100644
index 0000000..ab5f733
--- /dev/null
+++ b/office-translator-landing-page/components/waitlist-section.tsx
@@ -0,0 +1,141 @@
+"use client"
+
+import { useState, useCallback } from "react"
+import { Mail, Loader2, CheckCircle2, ArrowRight } from "lucide-react"
+import { track } from "@vercel/analytics"
+import { Button } from "@/components/ui/button"
+import { Input } from "@/components/ui/input"
+
+type Status = "idle" | "loading" | "success" | "error"
+
+// The backend is proxied under the same origin via the rewrites in
+// next.config.mjs, so the form never needs the backend origin (no CORS).
+const WAITLIST_ENDPOINT = "/api/v1/waitlist"
+
+const INTERESTS = [
+ "I translate documents regularly",
+ "I run a translation agency",
+ "I manage a multilingual team",
+ "I'm a developer (API integration)",
+ "Just exploring",
+]
+
+export function WaitlistSection() {
+ const [email, setEmail] = useState("")
+ const [interest, setInterest] = useState(INTERESTS[0])
+ const [status, setStatus] = useState("idle")
+ const [message, setMessage] = useState("")
+
+ const submit = useCallback(
+ async (e: React.FormEvent) => {
+ e.preventDefault()
+ if (!email || status === "loading") return
+ setStatus("loading")
+ setMessage("")
+ try {
+ const res = await fetch(`/api/v1/waitlist`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ email, interest }),
+ })
+ const data = await res.json().catch(() => ({}))
+ if (res.ok) {
+ setStatus("success")
+ track("waitlist_joined", { interest })
+ setMessage(
+ data?.data?.status === "already_joined"
+ ? "You are already on the list. We will email you at launch."
+ : "You are on the list. We will email you at launch."
+ )
+ setEmail("")
+ } else {
+ setStatus("error")
+ track("waitlist_error", { status: res.status })
+ setMessage(data?.message ?? "Something went wrong. Please try again.")
+ }
+ } catch {
+ setStatus("error")
+ setMessage("Network error. Please try again later.")
+ }
+ },
+ [email, interest, status]
+ )
+
+ return (
+
+
+
+
+
+
+
+ Be first in line at launch
+
+
+ Join the waitlist and get early access, launch-day pricing and the
+ “keep your format” playbook. No spam, one email at launch.
+
+
+
+ {status === "success" ? (
+
+ ) : (
+
+ )}
+
+
+ )
+}
diff --git a/office-translator-landing-page/lib/utils.ts b/office-translator-landing-page/lib/utils.ts
new file mode 100644
index 0000000..bd0c391
--- /dev/null
+++ b/office-translator-landing-page/lib/utils.ts
@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+ return twMerge(clsx(inputs))
+}
diff --git a/office-translator-landing-page/next-env.d.ts b/office-translator-landing-page/next-env.d.ts
new file mode 100644
index 0000000..9edff1c
--- /dev/null
+++ b/office-translator-landing-page/next-env.d.ts
@@ -0,0 +1,6 @@
+///
+///
+import "./.next/types/routes.d.ts";
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
diff --git a/office-translator-landing-page/next.config.mjs b/office-translator-landing-page/next.config.mjs
index 4cd9948..41e0315 100644
--- a/office-translator-landing-page/next.config.mjs
+++ b/office-translator-landing-page/next.config.mjs
@@ -1,4 +1,6 @@
/** @type {import('next').NextConfig} */
+const API_BASE = process.env.NEXT_PUBLIC_API_BASE_URL || "http://localhost:8000"
+
const nextConfig = {
typescript: {
ignoreBuildErrors: true,
@@ -6,6 +8,26 @@ const nextConfig = {
images: {
unoptimized: true,
},
+ async rewrites() {
+ return [
+ {
+ source: "/api/v1/waitlist/:path*",
+ destination: `${API_BASE}/api/v1/waitlist/:path*`,
+ },
+ {
+ source: "/api/v1/waitlist",
+ destination: `${API_BASE}/api/v1/waitlist`,
+ },
+ {
+ source: "/docs/:path*",
+ destination: `${API_BASE}/docs/:path*`,
+ },
+ {
+ source: "/openapi.json",
+ destination: `${API_BASE}/openapi.json`,
+ },
+ ]
+ },
}
export default nextConfig
diff --git a/office-translator-landing-page/tsconfig.json b/office-translator-landing-page/tsconfig.json
index 4b2dc7b..48d6d82 100644
--- a/office-translator-landing-page/tsconfig.json
+++ b/office-translator-landing-page/tsconfig.json
@@ -1,6 +1,10 @@
{
"compilerOptions": {
- "lib": ["dom", "dom.iterable", "esnext"],
+ "lib": [
+ "dom",
+ "dom.iterable",
+ "esnext"
+ ],
"allowJs": true,
"target": "ES6",
"skipLibCheck": true,
@@ -11,7 +15,7 @@
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
- "jsx": "preserve",
+ "jsx": "react-jsx",
"incremental": true,
"plugins": [
{
@@ -19,9 +23,19 @@
}
],
"paths": {
- "@/*": ["./*"]
+ "@/*": [
+ "./*"
+ ]
}
},
- "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
- "exclude": ["node_modules"]
+ "include": [
+ "next-env.d.ts",
+ "**/*.ts",
+ "**/*.tsx",
+ ".next/types/**/*.ts",
+ ".next/dev/types/**/*.ts"
+ ],
+ "exclude": [
+ "node_modules"
+ ]
}
diff --git a/requirements.txt b/requirements.txt
index 868f8b3..58522c2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-fastapi==0.109.0
+fastapi==0.109.1
uvicorn[standard]==0.27.0
-python-multipart==0.0.9
+python-multipart==0.0.20
openpyxl==3.1.2
python-docx==1.1.0
python-pptx==0.6.23
@@ -13,7 +13,7 @@ python-dotenv==1.0.0
pydantic==2.5.3
pydantic[email]==2.5.3
aiofiles==23.2.1
-httpx>=0.27.0
+httpx>=0.27.0,<0.28 # 0.28 removed Client(app=...) — breaks starlette TestClient
Pillow==10.2.0
matplotlib==3.8.2
pandas==2.1.4
diff --git a/routes/admin_routes.py b/routes/admin_routes.py
index 6af7af2..ce07c46 100644
--- a/routes/admin_routes.py
+++ b/routes/admin_routes.py
@@ -285,6 +285,67 @@ async def get_admin_dashboard(admin_id: str = Depends(require_admin)):
"last_check": None,
}
+ # OCR status (scanned PDFs): configured via admin settings or env key.
+ try:
+ from services.mistral_ocr import MistralOCRClient
+
+ settings = load_settings()
+ mistral_key = (settings.mistral.api_key or "").strip() or os.getenv(
+ "MISTRAL_API_KEY", ""
+ ).strip()
+ ocr_configured = bool(mistral_key) and (
+ settings.mistral.enabled
+ # enabled=False in a saved-but-untouched settings file must not
+ # hide an env-configured OCR (same rule as the translate route)
+ or not (settings.mistral.api_key or "").strip()
+ )
+ providers_status["mistral_ocr"] = {
+ "name": "mistral_ocr",
+ "available": ocr_configured,
+ "error": None
+ if ocr_configured
+ else "Non configuré — les PDF scannés seront refusés (MISTRAL_API_KEY)",
+ "last_check": None,
+ # No live call here: health is derived from configuration.
+ "config_only": True,
+ }
+ except Exception as e:
+ providers_status["mistral_ocr"] = {
+ "name": "mistral_ocr",
+ "available": False,
+ "error": str(e)[:100],
+ "last_check": None,
+ }
+
+ # LLM/translation engines configured via admin settings (key presence
+ # only — no live call, keys are never exposed).
+ try:
+ settings = load_settings()
+
+ def _engine_status(section_name: str, env_var: str, label: str):
+ section = getattr(settings, section_name, None)
+ key_set = bool(
+ ((getattr(section, "api_key", None) or "").strip())
+ or os.getenv(env_var, "").strip()
+ )
+ providers_status[section_name] = {
+ "name": section_name,
+ "available": key_set,
+ "error": None if key_set else f"Clé absente ({env_var})",
+ "last_check": None,
+ "config_only": True,
+ "label": label,
+ }
+
+ _engine_status("deepl", "DEEPL_API_KEY", "DeepL")
+ _engine_status("openrouter", "OPENROUTER_API_KEY", "Traduction IA Éco")
+ _engine_status("openrouter_premium", "OPENROUTER_API_KEY", "Traduction IA Premium")
+ _engine_status("openai", "OPENAI_API_KEY", "OpenAI")
+ _engine_status("zai", "ZAI_API_KEY", "Grok (xAI)")
+ _engine_status("google_cloud", "GOOGLE_CLOUD_API_KEY", "Google Cloud")
+ except Exception as e:
+ logger.warning(f"admin dashboard engines status failed: {e}")
+
return {
"timestamp": health_status.get("timestamp"),
"status": health_status.get("status"),
@@ -641,13 +702,24 @@ async def update_default_provider(
provider: str = Form(...),
admin_id: str = Depends(require_admin),
):
- """Update the default translation provider"""
+ """Update the default translation provider.
+
+ The allowed list mirrors the providers actually wired in the translate
+ route (and declared in SettingsConfig) so the admin can set any of them as
+ the default — previously google_cloud, openrouter_premium, deepseek and
+ minimax were wrongly rejected here even though they are fully supported.
+ """
valid_providers = [
"google",
+ "google_cloud",
"deepl",
"openai",
"openrouter",
+ "openrouter_premium",
+ "deepseek",
+ "minimax",
"zai",
+ # Mode aliases resolved at translation time.
"classic",
"llm",
]
@@ -852,6 +924,7 @@ class SettingsConfig(BaseModel):
deepseek: ProviderSettings = ProviderSettings()
minimax: ProviderSettings = ProviderSettings()
zai: ProviderSettings = ProviderSettings()
+ mistral: ProviderSettings = ProviderSettings() # OCR Mistral (PDF scannés)
smtp: SmtpSettings = SmtpSettings()
fallback_chain: str = "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
fallback_chain_classic: str = "google,google_cloud,deepl"
@@ -924,6 +997,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
payload["zai"] = _merge_env(settings.zai, key_env="ZAI_API_KEY", model_env="ZAI_MODEL", url_env="ZAI_BASE_URL", default_model="grok-2-1212", default_url="https://api.x.ai/v1")
payload["google_cloud"] = _merge_env(settings.google_cloud, key_env="GOOGLE_CLOUD_API_KEY")
+ payload["mistral"] = _merge_env(settings.mistral, key_env="MISTRAL_API_KEY", model_env="MISTRAL_OCR_MODEL", default_model="mistral-ocr-latest")
# SMTP: merge from env vars, but never expose password
smtp_data = settings.smtp.model_dump()
@@ -958,6 +1032,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
"zai": bool(os.getenv("ZAI_API_KEY", "").strip()),
"google_cloud": bool(os.getenv("GOOGLE_CLOUD_API_KEY", "").strip()),
+ "mistral": bool(os.getenv("MISTRAL_API_KEY", "").strip()),
"smtp": bool(os.getenv("SMTP_HOST", "").strip()),
}
return JSONResponse(
@@ -1029,6 +1104,49 @@ async def test_provider(
status_code=200, content={"available": True, "test_result": result}
)
+ elif provider == "mistral":
+ api_key = _key(provider_config.api_key, "MISTRAL_API_KEY")
+ if not api_key:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "available": False,
+ "error": "Aucune clé API Mistral trouvée (JSON ou .env MISTRAL_API_KEY).",
+ },
+ )
+ import requests as _requests
+
+ resp = _requests.get(
+ "https://api.mistral.ai/v1/models",
+ headers={"Authorization": f"Bearer {api_key}"},
+ timeout=10,
+ )
+ if resp.ok:
+ n_models = len(resp.json().get("data", []))
+ return JSONResponse(
+ status_code=200,
+ content={
+ "available": True,
+ "test_result": f"Clé valide — {n_models} modèles accessibles",
+ },
+ )
+ elif resp.status_code in (401, 403):
+ return JSONResponse(
+ status_code=resp.status_code,
+ content={
+ "available": False,
+ "error": f"Clé API Mistral invalide (HTTP {resp.status_code}).",
+ },
+ )
+ else:
+ return JSONResponse(
+ status_code=500,
+ content={
+ "available": False,
+ "error": f"Erreur Mistral HTTP {resp.status_code}: {resp.text[:200]}",
+ },
+ )
+
elif provider == "google_cloud":
api_key = _key(provider_config.api_key, "GOOGLE_CLOUD_API_KEY")
if not api_key:
diff --git a/routes/api_v1_router.py b/routes/api_v1_router.py
index 844c582..ef4cd52 100644
--- a/routes/api_v1_router.py
+++ b/routes/api_v1_router.py
@@ -15,6 +15,7 @@ from routes.admin_routes import router as admin_router
from routes.legacy_routes import router as legacy_router
from routes.glossary_routes import router as glossary_router
from routes.prompt_routes import router as prompt_router
+from routes.waitlist_routes import router as waitlist_router
router.include_router(translate_router, tags=["Translation"])
router.include_router(auth_router, tags=["Authentication"])
@@ -23,3 +24,4 @@ router.include_router(admin_router, tags=["Admin"])
router.include_router(legacy_router, tags=["Legacy"])
router.include_router(glossary_router, tags=["Glossaries"])
router.include_router(prompt_router, tags=["Prompts"])
+router.include_router(waitlist_router, tags=["Waitlist"])
diff --git a/routes/legacy_routes.py b/routes/legacy_routes.py
index 2161307..ff50efc 100644
--- a/routes/legacy_routes.py
+++ b/routes/legacy_routes.py
@@ -14,6 +14,7 @@ from fastapi.responses import FileResponse, JSONResponse
from config import config
from utils import file_handler
+from utils.file_handler import validate_zip_safety
from middleware.api_key_auth import get_authenticated_user
logger = logging.getLogger(__name__)
@@ -32,7 +33,9 @@ def _resolve_model(
@router.get("/providers/available")
-async def get_available_providers():
+async def get_available_providers(
+ current_user: Optional[Any] = Depends(get_authenticated_user),
+):
"""
Return every provider that is enabled — checking BOTH the admin settings JSON
AND environment variables (env vars act as a fallback / override).
@@ -42,8 +45,12 @@ async def get_available_providers():
- Ollama is only shown in DEV mode (APP_ENV=development or SHOW_OLLAMA=true).
- openrouter → shown as "Traduction IA Essentielle" (cheap models).
- openrouter_premium → shown as "Traduction IA Premium" (premium models).
+ - Filtered to the engines included in the caller's plan
+ (PLANS[plan]["providers"]) so the UI never offers an engine the
+ translate endpoint would reject.
"""
from routes.admin_routes import load_settings
+ from models.subscription import PlanType, PLANS
settings = load_settings()
is_dev = os.getenv("APP_ENV", "production").lower() == "development"
@@ -189,6 +196,16 @@ async def get_available_providers():
+ # Filter to the engines included in the caller's plan (anonymous → Free).
+ user_plan = PlanType.FREE
+ if current_user is not None:
+ try:
+ user_plan = PlanType(getattr(current_user, "plan", PlanType.FREE))
+ except ValueError:
+ user_plan = PlanType.FREE
+ allowed = set((PLANS.get(user_plan) or PLANS[PlanType.FREE]).get("providers", []))
+ available = [p for p in available if p.get("id") in allowed]
+
return JSONResponse(
status_code=200,
headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
@@ -198,49 +215,36 @@ async def get_available_providers():
@router.get("/languages")
async def get_supported_languages():
- """Get list of supported language codes, ordered by internet popularity"""
+ """Get list of supported language codes, ordered by internet popularity.
+
+ Served from LanguageValidator (single source of truth): the 35 most
+ requested languages first, then the rest of the validated ISO 639-1 set
+ alphabetically. "auto" is excluded — it is a source-only value handled
+ by the source_lang parameter.
+ """
+ from middleware.validation import LanguageValidator
+
+ popular_order = [
+ # Top 5 — dominant on the internet
+ "en", "es", "de", "fr", "ja",
+ # Top 6-15
+ "pt", "ru", "it", "zh-CN", "zh-TW", "pl", "nl", "tr", "ko", "ar",
+ # Top 16-25
+ "fa", "vi", "id", "uk", "sv", "cs", "el", "he", "hi", "ro",
+ # Next most requested
+ "da", "fi", "no", "hu", "th", "sk", "bg", "hr", "ca", "ms", "zh",
+ ]
+
+ names = LanguageValidator.LANGUAGE_NAMES
+ supported = [
+ c for c in LanguageValidator.SUPPORTED_LANGUAGES if c != "auto"
+ ]
+ ordered = [c for c in popular_order if c in supported]
+ ordered += sorted(c for c in supported if c not in ordered)
+
return {
- "supported_languages": {
- # Top 5 — dominant on the internet
- "en": "English",
- "es": "Spanish",
- "de": "German",
- "fr": "French",
- "ja": "Japanese",
- # Top 6-15
- "pt": "Portuguese",
- "ru": "Russian",
- "it": "Italian",
- "zh-CN": "Chinese (Simplified)",
- "zh-TW": "Chinese (Traditional)",
- "pl": "Polish",
- "nl": "Dutch",
- "tr": "Turkish",
- "ko": "Korean",
- "ar": "Arabic",
- # Top 16-25
- "fa": "Persian (Farsi)",
- "vi": "Vietnamese",
- "id": "Indonesian",
- "uk": "Ukrainian",
- "sv": "Swedish",
- "cs": "Czech",
- "el": "Greek",
- "he": "Hebrew",
- "hi": "Hindi",
- "ro": "Romanian",
- # Others
- "da": "Danish",
- "fi": "Finnish",
- "no": "Norwegian",
- "hu": "Hungarian",
- "th": "Thai",
- "sk": "Slovak",
- "bg": "Bulgarian",
- "hr": "Croatian",
- "ca": "Catalan",
- "ms": "Malay",
- },
+ "supported_languages": {code: names.get(code, code.upper()) for code in ordered},
+ "count": len(ordered),
"note": "Supported languages may vary depending on the translation service configured",
}
@@ -276,6 +280,10 @@ async def translate_batch_documents(
file_handler.save_upload_file(file, input_path)
+ # Zip bomb protection: Office files are ZIP archives
+ if file_extension != ".pdf":
+ validate_zip_safety(input_path)
+
if file_extension == ".xlsx":
excel_translator.translate_file(
input_path, output_path, target_language, source_language
@@ -300,6 +308,18 @@ async def translate_batch_documents(
}
)
+ except ValueError as e:
+ file_handler.cleanup_file(input_path)
+ logger.warning(f"Rejected unsafe or invalid archive: {file.filename}: {e}")
+ results.append(
+ {
+ "filename": file.filename,
+ "status": "error",
+ "error": "CORRUPTED_FILE",
+ "message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
+ "details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
+ }
+ )
except Exception as e:
logger.exception(f"Error processing {file.filename}")
results.append(
@@ -341,6 +361,9 @@ async def extract_texts_from_document(
input_path = config.UPLOAD_DIR / input_filename
file_handler.save_upload_file(file, input_path)
+ if file_extension != ".pdf":
+ validate_zip_safety(input_path)
+
texts = []
if file_extension == ".xlsx":
@@ -423,6 +446,17 @@ async def extract_texts_from_document(
except HTTPException:
raise
+ except ValueError as e:
+ file_handler.cleanup_file(input_path)
+ logger.warning(f"Text extraction rejected unsafe or invalid archive: {file.filename}: {e}")
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": "CORRUPTED_FILE",
+ "message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
+ "details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
+ },
+ )
except Exception as e:
logger.exception("Text extraction error")
return JSONResponse(
diff --git a/routes/translate_routes.py b/routes/translate_routes.py
index 2539bdf..9a0a38b 100644
--- a/routes/translate_routes.py
+++ b/routes/translate_routes.py
@@ -9,6 +9,7 @@ Story 3.6: Documentation OpenAPI complète avec exemples et codes d'erreur
import os
import re
+import secrets
import uuid
import time
import socket
@@ -43,7 +44,7 @@ from typing_extensions import Annotated
from config import config
from translators import ExcelTranslator, WordTranslator, PowerPointTranslator
-from models.subscription import PlanType
+from models.subscription import PlanType, PLANS
from services.auth_service import (
record_usage,
check_usage_limits,
@@ -54,6 +55,7 @@ from middleware.tier_quota import _seconds_until_next_month, _next_month_utc
from middleware.validation import FileValidator, ValidationError, LanguageValidator, webhook_validator
from middleware.api_key_auth import get_authenticated_user, get_user_from_api_key
from utils import file_handler
+from utils.file_handler import validate_zip_safety
# Import models from schemas (Story 3.6 - DRY principle)
from schemas.translation import (
@@ -148,6 +150,43 @@ def _tier_for_quota(plan) -> str:
return "free"
+def _tier_from_plan_str(plan_str: Optional[str]) -> str:
+ """Map the job's ``user_plan`` string to a quota tier.
+
+ ``user_plan`` is built with ``str(current_user.plan)`` which renders a
+ str-Enum as "PlanType.PRO" — accept both that repr and the raw value.
+ """
+ name = (plan_str or "").strip().lower()
+ if name.startswith("plantype."):
+ name = name.split(".", 1)[1]
+ try:
+ return _tier_for_quota(PlanType(name))
+ except ValueError:
+ return "free"
+
+
+def _plan_from_user(current_user: Optional[Any]) -> PlanType:
+ """Resolve the request's plan (anonymous requests are on the Free plan)."""
+ if current_user is None:
+ return PlanType.FREE
+ plan = getattr(current_user, "plan", PlanType.FREE)
+ try:
+ return PlanType(plan)
+ except ValueError:
+ return PlanType.FREE
+
+
+def _allowed_providers_for_plan(plan: PlanType) -> set:
+ """Engines included in the plan (source of truth: PLANS[plan]["providers"])."""
+ plan_cfg = PLANS.get(plan) or PLANS[PlanType.FREE]
+ return set(plan_cfg.get("providers", []))
+
+
+def _image_translation_allowed_for_plan(plan: PlanType) -> bool:
+ """Vision (text-in-images) is sold on Pro and above."""
+ return plan in (PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE)
+
+
def _next_midnight_utc() -> datetime:
"""Get next midnight UTC."""
now = datetime.now(timezone.utc)
@@ -215,6 +254,21 @@ def _parse_content_disposition(content_disp: str) -> Optional[str]:
return None
+def _sanitize_url_filename(filename: str) -> str:
+ """Strip path traversal / control chars from a remote-provided filename."""
+ filename = Path(filename).name
+ filename = re.sub(r"[\x00-\x1f\x7f-\x9f]", "", filename)
+ filename = re.sub(r'[<>:"/\\|?*]', "_", filename)
+ # Drop leading dots (hidden files, ".." leftovers)
+ filename = filename.lstrip(".")
+ if len(filename) > 255:
+ name, ext = (
+ filename.rsplit(".", 1) if "." in filename else (filename, "")
+ )
+ filename = name[:250] + ("." + ext if ext else "")
+ return filename or "downloaded_file"
+
+
def _is_ssrf_risk(hostname: str) -> bool:
"""Return True if hostname resolves to a private/reserved IP (SSRF prevention).
@@ -252,95 +306,130 @@ async def download_from_url(url: str, timeout: int = 30) -> tuple[Path, str]:
details={"scheme": parsed_url.scheme or "none"},
)
- hostname = parsed_url.hostname or ""
- if not hostname or _is_ssrf_risk(hostname):
- raise TranslateEndpointError(
- code=TranslateEndpointError.URL_UNREACHABLE,
- message="The URL points to a blocked address (private or internal network).",
- details={"reason": "ssrf_blocked"},
- )
+ def _validate_url_target(u: str):
+ p = urlparse(u)
+ if p.scheme not in ("http", "https"):
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.URL_UNREACHABLE,
+ message="Only HTTP/HTTPS URLs are accepted.",
+ details={"scheme": p.scheme or "none"},
+ )
+ h = p.hostname or ""
+ if not h or _is_ssrf_risk(h):
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.URL_UNREACHABLE,
+ message="The URL points to a blocked address (private or internal network).",
+ details={"reason": "ssrf_blocked"},
+ )
+
+ _validate_url_target(url)
+
+ MAX_REDIRECTS = 5
try:
async with httpx.AsyncClient(
- timeout=timeout, follow_redirects=True, max_redirects=10
+ timeout=timeout, follow_redirects=False
) as client:
- async with client.stream("GET", url) as response:
- if response.status_code != 200:
- raise TranslateEndpointError(
- code=TranslateEndpointError.URL_UNREACHABLE,
- message=f"URL unreachable (HTTP {response.status_code})",
- details={"status_code": response.status_code, "url": url[:100]},
- )
-
- content_length = response.headers.get("content-length")
- if content_length:
- try:
- file_size = int(content_length)
- max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
- if file_size > max_size_bytes:
+ current_url = url
+ for _ in range(MAX_REDIRECTS + 1):
+ _validate_url_target(current_url)
+ async with client.stream("GET", current_url) as response:
+ if response.status_code in (301, 302, 303, 307, 308):
+ location = response.headers.get("location", "")
+ if not location:
raise TranslateEndpointError(
- code=TranslateEndpointError.FILE_TOO_LARGE,
- message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
- details={
- "size_mb": round(file_size / (1024 * 1024), 2),
- "max_mb": MAX_FILE_SIZE_MB,
- },
+ code=TranslateEndpointError.URL_UNREACHABLE,
+ message="Redirect without Location header.",
+ details={"reason": "invalid_redirect"},
)
- except ValueError:
- pass
+ # Re-check the redirect target before following it
+ current_url = str(httpx.URL(current_url).join(location))
+ continue
- filename = None
- content_disp = response.headers.get("content-disposition", "")
- if content_disp:
- filename = _parse_content_disposition(content_disp)
+ if response.status_code != 200:
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.URL_UNREACHABLE,
+ message=f"URL unreachable (HTTP {response.status_code})",
+ details={"status_code": response.status_code, "url": url[:100]},
+ )
- if not filename:
- filename = unquote(Path(parsed_url.path).name) or "downloaded_file"
+ content_length = response.headers.get("content-length")
+ if content_length:
+ try:
+ file_size = int(content_length)
+ max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
+ if file_size > max_size_bytes:
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.FILE_TOO_LARGE,
+ message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
+ details={
+ "size_mb": round(file_size / (1024 * 1024), 2),
+ "max_mb": MAX_FILE_SIZE_MB,
+ },
+ )
+ except ValueError:
+ pass
- extension = Path(filename).suffix.lower()
- if extension not in ACCEPTED_EXTENSIONS:
- raise TranslateEndpointError(
- code=TranslateEndpointError.INVALID_FORMAT,
- details={
- "detected_extension": extension or "none",
- "accepted_formats": list(ACCEPTED_EXTENSIONS),
- },
- )
+ filename = None
+ content_disp = response.headers.get("content-disposition", "")
+ if content_disp:
+ filename = _parse_content_disposition(content_disp)
- unique_id = str(uuid.uuid4())[:8]
- safe_filename = f"{unique_id}_{filename}"
- temp_path = config.UPLOAD_DIR / safe_filename
+ if not filename:
+ filename = unquote(Path(urlparse(current_url).path).name)
- temp_path.parent.mkdir(parents=True, exist_ok=True)
+ filename = _sanitize_url_filename(filename)
- max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
- downloaded_bytes = 0
+ extension = Path(filename).suffix.lower()
+ if extension not in ACCEPTED_EXTENSIONS:
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.INVALID_FORMAT,
+ details={
+ "detected_extension": extension or "none",
+ "accepted_formats": list(ACCEPTED_EXTENSIONS),
+ },
+ )
- async with aiofiles.open(temp_path, "wb") as f:
- async for chunk in response.aiter_bytes(chunk_size=65536):
- downloaded_bytes += len(chunk)
+ unique_id = str(uuid.uuid4())[:8]
+ safe_filename = f"{unique_id}_{filename}"
+ temp_path = config.UPLOAD_DIR / safe_filename
- if downloaded_bytes > max_size_bytes:
- await f.close()
- if temp_path.exists():
- temp_path.unlink()
- raise TranslateEndpointError(
- code=TranslateEndpointError.FILE_TOO_LARGE,
- details={
- "size_mb": round(
- downloaded_bytes / (1024 * 1024), 2
- ),
- "max_mb": MAX_FILE_SIZE_MB,
- },
- )
+ temp_path.parent.mkdir(parents=True, exist_ok=True)
- await f.write(chunk)
+ max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
+ downloaded_bytes = 0
- async with aiofiles.open(temp_path, "rb") as f:
- header = await f.read(4)
- await validate_file_content(header, extension)
+ async with aiofiles.open(temp_path, "wb") as f:
+ async for chunk in response.aiter_bytes(chunk_size=65536):
+ downloaded_bytes += len(chunk)
- return temp_path, filename
+ if downloaded_bytes > max_size_bytes:
+ await f.close()
+ if temp_path.exists():
+ temp_path.unlink()
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.FILE_TOO_LARGE,
+ details={
+ "size_mb": round(
+ downloaded_bytes / (1024 * 1024), 2
+ ),
+ "max_mb": MAX_FILE_SIZE_MB,
+ },
+ )
+
+ await f.write(chunk)
+
+ async with aiofiles.open(temp_path, "rb") as f:
+ header = await f.read(4)
+ await validate_file_content(header, extension)
+
+ return temp_path, filename
+
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.URL_UNREACHABLE,
+ message="Too many redirects.",
+ details={"reason": "too_many_redirects", "max_redirects": MAX_REDIRECTS},
+ )
except httpx.TimeoutException:
if temp_path and temp_path.exists():
@@ -420,9 +509,13 @@ def _cleanup_old_jobs() -> None:
return
_last_cleanup_ts = current_time
+ # Snapshot the items before filtering: concurrent coroutines (background
+ # translation workers, status pollers) mutate _translation_jobs, and
+ # iterating a dict while it is resized raises
+ # "RuntimeError: dictionary changed size during iteration".
expired_job_ids = [
job_id
- for job_id, job in _translation_jobs.items()
+ for job_id, job in list(_translation_jobs.items())
if job.get("status") in ("completed", "failed")
and (
(ts := job.get("completed_at") or job.get("failed_at"))
@@ -444,6 +537,68 @@ def _job_age_seconds(timestamp_str: str) -> float:
return 0.0
+def _provider_model(provider: Any) -> str:
+ """Best-effort read of a provider's model name.
+
+ The new-style providers (``services/providers/*``) store the model in
+ ``self._model``, while the legacy providers (``services/translation_service``)
+ expose ``self.model``. Read either so cost-factor logic works for both.
+ """
+ if provider is None:
+ return ""
+ return (
+ getattr(provider, "_model", None)
+ or getattr(provider, "model", None)
+ or ""
+ )
+
+
+def _compute_cost_factor(provider: Any, provider_name: str = "") -> int:
+ """Billing cost factor (1 = standard, 5 = premium).
+
+ Premium models (Claude, GPT-4 family, etc.) cost more and are billed at a
+ higher factor. Cheap variants are explicitly downgraded to 1 (``haiku``,
+ and the small GPT-4 models such as ``gpt-4o-mini`` / ``gpt-4o-nano``).
+ The provider may be passed by instance (model read off it) or only by
+ name (e.g. the ``openrouter_premium`` alias).
+ """
+ model_lower = _provider_model(provider).lower()
+ provider_lower = (provider_name or "").lower()
+
+ if "haiku" in model_lower or "mini" in model_lower or "nano" in model_lower:
+ return 1
+ if any(k in model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
+ return 5
+ return 1
+
+
+def _compute_duration_seconds(created_at_iso: str) -> float:
+ """Elapsed seconds since ``created_at`` (UTC ISO string).
+
+ Uses a timezone-aware timestamp() to avoid the local-time bug that
+ ``time.mktime`` introduced. Falls back to 0 on parse errors so a bad
+ timestamp can never flip a successful job into the error branch.
+ """
+ return _job_age_seconds(created_at_iso)
+
+
+async def _release_quota_if_needed(user_id, usage_recorded: bool, job_id: str) -> None:
+ """Release the reserved translation quota on a soft-failure path.
+
+ The translation worker reserves a quota slot at request time. The generic
+ ``except`` branch releases it on hard failures, but the early ``return``
+ paths (empty output / no translatable text / 0 texts translated) are NOT
+ exceptions and must release the quota explicitly — otherwise the user
+ loses a slot without receiving a translation.
+ """
+ if user_id and not usage_recorded:
+ try:
+ await asyncio.to_thread(release_translation_quota, user_id)
+ logger.info(f"Job {job_id}: released reserved quota after soft-failure")
+ except Exception as release_err:
+ logger.exception(f"Job {job_id}: failed to release reserved quota: {release_err}")
+
+
@router_v1.post(
"/translate",
response_model=TranslateResponse,
@@ -476,6 +631,14 @@ async def translate_document_v1(
pdf_mode: Optional[Literal["layout", "text_only"]] = Form(
default=None, description="PDF translation mode: 'layout' (preserve layout) or 'text_only' (clean text output). PDF only."
),
+ formality: Optional[Literal["formal", "informal"]] = Form(
+ default=None,
+ description="Tone override for LLM engines: 'formal' or 'informal'. Ignored by classic engines.",
+ ),
+ output_mode: Optional[Literal["single", "bilingual"]] = Form(
+ default="single",
+ description="Output mode: 'single' (translated file only) or 'bilingual' (docx with source + translation interleaved).",
+ ),
translate_images: bool = Form(
default=False, description="Translate text inside images using AI vision"
),
@@ -676,7 +839,9 @@ async def translate_document_v1(
rate_limit_remaining = -1
try:
- LanguageValidator.validate(target_lang)
+ # Keep the canonical form (e.g. "zh-CN") so providers receive a
+ # code they understand instead of the raw user input.
+ target_lang = LanguageValidator.validate(target_lang)
except ValidationError as e:
raise TranslateEndpointError(
code="INVALID_FORMAT",
@@ -686,7 +851,7 @@ async def translate_document_v1(
if source_lang and source_lang != "auto":
try:
- LanguageValidator.validate(source_lang)
+ source_lang = LanguageValidator.validate(source_lang)
except ValidationError:
raise TranslateEndpointError(
code="INVALID_FORMAT",
@@ -760,7 +925,23 @@ async def translate_document_v1(
details={"error": "sha256_calculation_failed"},
)
+ # Office files are ZIP archives: reject archives that expand far
+ # beyond their uploaded size (zip bomb protection).
+ if file_extension != ".pdf":
+ try:
+ validate_zip_safety(input_path)
+ except ValueError as e:
+ file_handler_util.cleanup_file(input_path)
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.CORRUPTED_FILE,
+ message="The file is invalid or expands dangerously.",
+ details={"reason": "unsafe_archive", "detail": str(e)[:200]},
+ )
+
job_id = f"tr_{uuid.uuid4().hex[:12]}"
+ # Secret per-job token: required to follow/download a job that has no
+ # logged-in owner (anonymous API calls).
+ job_access_token = secrets.token_urlsafe(24)
# Track file metadata in Redis with TTL
await storage_tracker.track_file(
@@ -771,6 +952,7 @@ async def translate_document_v1(
"file_hash": file_hash,
"input_path": str(input_path),
"user_id": str(user_id) if user_id else None,
+ "access_token": job_access_token,
"timestamp": datetime.now(timezone.utc).isoformat(),
},
)
@@ -794,6 +976,7 @@ async def translate_document_v1(
"target_lang": target_lang,
"created_at": datetime.now(timezone.utc).isoformat(),
"user_id": user_id,
+ "access_token": job_access_token,
"input_path": str(input_path),
"file_extension": file_extension,
"provider": provider or mode,
@@ -803,6 +986,8 @@ async def translate_document_v1(
"prompt_id": prompt_id, # Story 3.12: Store prompt_id
"pdf_mode": pdf_mode, # PDF translation mode
"translate_images": translate_images,
+ "formality": formality, # LLM tone override
+ "output_mode": output_mode or "single", # single | bilingual
}
await set_job_status_async(job_id, _translation_jobs[job_id])
@@ -821,6 +1006,39 @@ async def translate_document_v1(
)
provider_to_use = "google"
+ # ── Plan-based engine gating (grille commerciale : PLANS[plan]["providers"]) ──
+ # Chaque plan n'expose que ses moteurs ; un moteur hors plan est refusé
+ # (403) au lieu d'être exécuté aux frais de la maison.
+ _user_plan = _plan_from_user(current_user)
+ if provider_to_use not in _allowed_providers_for_plan(_user_plan):
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
+ message=(
+ f"Le moteur « {provider_to_use} » n'est pas inclus dans votre plan. "
+ "Passez à un plan supérieur pour l'utiliser."
+ ),
+ details={
+ "feature": "provider",
+ "provider": provider_to_use,
+ "plan": str(_user_plan.value),
+ "allowed_providers": sorted(_allowed_providers_for_plan(_user_plan)),
+ },
+ )
+
+ # La traduction du texte dans les images (vision) est vendue Pro et plus.
+ if translate_images and not _image_translation_allowed_for_plan(_user_plan):
+ raise TranslateEndpointError(
+ code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
+ message=(
+ "La traduction du texte dans les images est réservée "
+ "aux plans Pro et supérieurs."
+ ),
+ details={
+ "feature": "translate_images",
+ "plan": str(_user_plan.value),
+ },
+ )
+
asyncio.create_task(
_run_translation_job(
job_id=job_id,
@@ -837,6 +1055,8 @@ async def translate_document_v1(
user_plan=str(current_user.plan) if current_user else "free",
pdf_mode=pdf_mode,
translate_images=translate_images,
+ formality=formality,
+ output_mode=output_mode or "single",
)
)
@@ -853,6 +1073,8 @@ async def translate_document_v1(
"file_name": original_filename,
"source_lang": source_lang,
"target_lang": target_lang,
+ # Needed to poll/download this job when calling without login
+ "access_token": job_access_token,
},
"meta": {
"rate_limit_remaining": rate_limit_remaining,
@@ -980,6 +1202,8 @@ async def _run_translation_job(
user_plan: Optional[str] = None, # Plan name for watermark decision
pdf_mode: Optional[str] = None, # PDF translation mode: "layout" or "text_only"
translate_images: bool = False,
+ formality: Optional[str] = None, # LLM tone override: formal/informal
+ output_mode: str = "single", # single | bilingual
) -> None:
"""
Run translation job in background with progress tracking.
@@ -1076,11 +1300,13 @@ async def _run_translation_job(
# Use custom_prompt if no prompt_id
effective_prompt = custom_prompt
- # Build the full prompt combining effective prompt and glossary
+ # Build the full prompt combining effective prompt, glossary,
+ # formality directive and regional variant hint.
full_prompt = build_full_prompt(
effective_prompt, glossary_terms,
source_lang=glossary_source_lang, target_lang=target_lang,
glossary_target_lang=glossary_target_lang,
+ formality=formality,
)
from services.providers.google_provider import GoogleTranslationProvider
@@ -1259,6 +1485,8 @@ async def _run_translation_job(
job_translator = ExcelTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
+ if hasattr(job_translator, "set_tm_scope"):
+ job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1272,6 +1500,8 @@ async def _run_translation_job(
job_translator = WordTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
+ if hasattr(job_translator, "set_tm_scope"):
+ job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1285,6 +1515,8 @@ async def _run_translation_job(
job_translator = PowerPointTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
+ if hasattr(job_translator, "set_tm_scope"):
+ job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1299,6 +1531,40 @@ async def _run_translation_job(
job_translator = PDFTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
+ if hasattr(job_translator, "set_tm_scope"):
+ job_translator.set_tm_scope(user_id, full_prompt)
+ # OCR (PDF scannés) : réglages admin > variables d'env.
+ mistral_cfg = getattr(_admin_cfg, "mistral", None)
+ mistral_key = _cfg(
+ getattr(mistral_cfg, "api_key", None), "MISTRAL_API_KEY"
+ )
+ # Only honor the admin "enabled" flag when Mistral is actually
+ # configured there — a freshly saved settings file carries
+ # enabled=false defaults that must not override env-based OCR.
+ mistral_admin_configured = bool(
+ mistral_cfg is not None
+ and (getattr(mistral_cfg, "api_key", None) or "").strip()
+ )
+ job_translator.set_ocr_config(
+ api_key=mistral_key,
+ model=_cfg(
+ getattr(mistral_cfg, "model", None),
+ "MISTRAL_OCR_MODEL",
+ "mistral-ocr-latest",
+ ),
+ timeout=int(
+ _cfg(
+ str(getattr(mistral_cfg, "timeout", "") or ""),
+ "MISTRAL_OCR_TIMEOUT",
+ "180",
+ )
+ ),
+ enabled=(
+ getattr(mistral_cfg, "enabled", None)
+ if mistral_admin_configured
+ else None
+ ),
+ )
actual_output = await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1320,19 +1586,21 @@ async def _run_translation_job(
error_msg = "Translation failed: output file is empty or missing. The translation provider may be unavailable."
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
+ await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
stats = job_translator.get_translation_stats()
attempted = stats.get("attempted", 0)
changed = stats.get("changed", 0)
- if attempted == 0 and file_extension in ('.docx', '.xlsx', '.pptx'):
+ if attempted == 0:
error_msg = (
"Aucun texte traduisible détecté dans le document. "
"Le fichier est peut-être vide, protégé, ou ne contient que des images."
)
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
+ await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
if attempted > 0:
@@ -1346,6 +1614,7 @@ async def _run_translation_job(
)
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
+ await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
elif ratio < 0.05:
# Very suspicious — likely partial failure, warn but don't block
@@ -1430,9 +1699,7 @@ async def _run_translation_job(
from middleware.metrics import record_translation_retry
record_translation_retry(
reason="l1_fail",
- tier=_tier_for_quota(
- current_user.plan if current_user else None
- ) if current_user else "free",
+ tier=_tier_from_plan_str(user_plan),
)
except Exception:
pass
@@ -1448,9 +1715,7 @@ async def _run_translation_job(
from middleware.metrics import record_translation_retry
record_translation_retry(
reason="l0_fail",
- tier=_tier_for_quota(
- current_user.plan if current_user else None
- ) if current_user else "free",
+ tier=_tier_from_plan_str(user_plan),
)
except Exception:
pass
@@ -1467,9 +1732,7 @@ async def _run_translation_job(
from services.quality import run_l2_check
# Tier gate: Pro+ plans only (unless gate is disabled)
- user_tier = (
- _tier_for_quota(current_user.plan) if current_user else "free"
- )
+ user_tier = _tier_from_plan_str(user_plan)
tier_gate_on = getattr(config, "QUALITY_L2_TIER_GATE", True)
if not tier_gate_on or user_tier in ("pro", "business", "enterprise"):
translated_chunks_for_l2 = [s["translated"] for s in quality_samples]
@@ -1498,21 +1761,52 @@ async def _run_translation_job(
f"Job {job_id}: quality L2 layer failed: {l2_err}"
)
- if user_id:
- # Determine cost factor based on selected provider and model
- cost_factor = 1
- provider_lower = (provider or "").lower()
-
- prov_model = ""
- if translation_provider:
- prov_model = getattr(translation_provider, "model", "") or ""
-
- prov_model_lower = prov_model.lower()
- if any(k in prov_model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
- if "haiku" in prov_model_lower:
- cost_factor = 1
+ # ------------------------------------------------------------------
+ # QA report (pure heuristics — numbers fidelity, untranslated
+ # ratio, 0-100 score). Never blocks the job; surfaced in the job
+ # status for the UI/API.
+ # ------------------------------------------------------------------
+ try:
+ from services.quality.qa_report import run_qa_report
+
+ qa = await asyncio.to_thread(
+ run_qa_report, input_path, output_path, target_lang, file_extension
+ )
+ if qa:
+ job["quality"] = qa
+ except Exception as qa_err:
+ logger.warning(f"Job {job_id}: QA report failed: {qa_err}")
+
+ # ------------------------------------------------------------------
+ # Bilingual output (docx): interleaves source paragraphs above
+ # their translation. Falls back silently to the translated file.
+ # ------------------------------------------------------------------
+ if output_mode == "bilingual" and file_extension == ".docx":
+ try:
+ from translators.bilingual import make_bilingual_docx
+
+ bilingual_path = output_path.with_name(
+ output_path.stem + "_bilingual" + output_path.suffix
+ )
+ actual = await asyncio.to_thread(
+ make_bilingual_docx, input_path, output_path, bilingual_path
+ )
+ if actual:
+ output_path = Path(actual)
+ logger.info(f"Job {job_id}: bilingual output generated")
else:
- cost_factor = 5
+ logger.warning(
+ f"Job {job_id}: bilingual output skipped "
+ "(structure mismatch) — returning translated file"
+ )
+ except Exception as bi_err:
+ logger.warning(f"Job {job_id}: bilingual output failed: {bi_err}")
+
+ if user_id:
+ # Determine cost factor based on selected provider and model.
+ # _compute_cost_factor reads the model off the provider robustly
+ # (new-style providers store it in ``_model``, legacy in ``model``).
+ cost_factor = _compute_cost_factor(translation_provider, provider or "")
# Persist monthly usage counters in PostgreSQL (docs + pages)
pages = await asyncio.to_thread(
@@ -1537,7 +1831,7 @@ async def _run_translation_job(
tracker.set_completed(str(output_path))
# Record translation metric
- duration = time.time() - time.mktime(datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).timetuple())
+ duration = _compute_duration_seconds(job.get("created_at", ""))
record_translation(provider=provider, file_type=file_extension or "unknown", duration=duration, status="success")
logger.info(f"Job {job_id}: Completed successfully")
@@ -1621,6 +1915,50 @@ async def _run_translation_job(
)
+def _check_job_access(
+ job: dict, current_user, token: Optional[str]
+) -> Optional[JSONResponse]:
+ """Return an error response if the caller may not see this job, else None.
+
+ Jobs owned by a logged-in user require that same user. Anonymous jobs
+ require the secret per-job token returned when the job was created.
+ """
+ job_user_id = job.get("user_id")
+ if job_user_id:
+ if not current_user:
+ return JSONResponse(
+ status_code=401,
+ content={
+ "error": "AUTH_REQUIRED",
+ "message": "Authentication is required to access this job.",
+ "details": {"job_id": job.get("id")},
+ },
+ )
+ if str(job_user_id) != str(current_user.id):
+ return JSONResponse(
+ status_code=403,
+ content={
+ "error": "ACCESS_DENIED",
+ "message": "You do not have access to this job.",
+ "details": {"job_id": job.get("id")},
+ },
+ )
+ return None
+
+ expected = job.get("access_token")
+ provided = token or ""
+ if not expected or not secrets.compare_digest(provided, expected):
+ return JSONResponse(
+ status_code=403,
+ content={
+ "error": "ACCESS_DENIED",
+ "message": "You do not have access to this job.",
+ "details": {"job_id": job.get("id"), "hint": "token_required"},
+ },
+ )
+ return None
+
+
@router_v1.get(
"/translations/{job_id}",
response_model=TranslationStatusResponse,
@@ -1631,6 +1969,7 @@ async def _run_translation_job(
)
async def get_translation_status(
job_id: str,
+ token: Optional[str] = None,
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
@@ -1680,6 +2019,10 @@ async def get_translation_status(
},
)
+ denied = _check_job_access(job, current_user, token)
+ if denied:
+ return denied
+
response_data = {
"id": job["id"],
"status": job["status"],
@@ -1689,6 +2032,7 @@ async def get_translation_status(
"source_lang": job.get("source_lang"),
"target_lang": job.get("target_lang"),
"created_at": job.get("created_at"),
+ "quality": job.get("quality"),
}
estimated_remaining = None
@@ -1768,6 +2112,7 @@ def _cleanup_files(input_path: Optional[str], output_path: Optional[str]) -> Non
)
async def download_translated_file(
job_id: str,
+ token: Optional[str] = None,
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
@@ -1819,16 +2164,9 @@ async def download_translated_file(
},
)
- job_user_id = job.get("user_id")
- if current_user and job_user_id and str(job_user_id) != str(current_user.id):
- return JSONResponse(
- status_code=403,
- content={
- "error": "ACCESS_DENIED",
- "message": "You do not have access to this file.",
- "details": {"job_id": job_id},
- },
- )
+ denied = _check_job_access(job, current_user, token)
+ if denied:
+ return denied
if job.get("status") != "completed":
return JSONResponse(
diff --git a/routes/waitlist_routes.py b/routes/waitlist_routes.py
new file mode 100644
index 0000000..1333e4f
--- /dev/null
+++ b/routes/waitlist_routes.py
@@ -0,0 +1,98 @@
+"""
+Waitlist / email-capture routes for the marketing landing page.
+
+Public endpoint: POST /api/v1/waitlist (no auth) — deduplicated by email,
+persisted in data/waitlist.json (same JSON-file pattern as provider settings).
+"""
+
+import json
+import logging
+import threading
+from datetime import datetime, timezone
+from typing import List, Optional
+
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, EmailStr
+
+from config import config
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/waitlist", tags=["Waitlist"])
+
+WAITLIST_FILE = config.BASE_DIR / "data" / "waitlist.json"
+_write_lock = threading.Lock()
+
+
+class WaitlistEntry(BaseModel):
+ email: EmailStr
+ interest: Optional[str] = None
+
+
+def _load_waitlist() -> List[dict]:
+ if not WAITLIST_FILE.exists():
+ return []
+ try:
+ with open(WAITLIST_FILE, encoding="utf-8") as f:
+ data = json.load(f)
+ return data if isinstance(data, list) else []
+ except Exception as e:
+ logger.warning(f"Failed to load waitlist: {e}")
+ return []
+
+
+def _save_waitlist(entries: List[dict]) -> None:
+ WAITLIST_FILE.parent.mkdir(parents=True, exist_ok=True)
+ with open(WAITLIST_FILE, "w", encoding="utf-8") as f:
+ json.dump(entries, f, indent=2)
+
+
+@router.post("", status_code=201, summary="Join the waitlist")
+async def join_waitlist(entry: WaitlistEntry):
+ email = entry.email.lower().strip()
+ now = datetime.now(timezone.utc).isoformat()
+
+ with _write_lock:
+ entries = _load_waitlist()
+ for existing in entries:
+ if existing.get("email", "").lower() == email:
+ existing["updated_at"] = now
+ if entry.interest:
+ existing["interest"] = entry.interest
+ _save_waitlist(entries)
+ logger.info(f"Waitlist rejoin: {email}")
+ return JSONResponse(
+ status_code=200,
+ content={
+ "data": {"email": email, "status": "already_joined"},
+ "message": "You are already on the list.",
+ },
+ )
+
+ entries.append(
+ {
+ "email": email,
+ "interest": entry.interest,
+ "joined_at": now,
+ "updated_at": now,
+ }
+ )
+ _save_waitlist(entries)
+ logger.info(f"Waitlist join: {email} (total={len(entries)})")
+
+ return JSONResponse(
+ status_code=201,
+ content={
+ "data": {"email": email, "status": "joined"},
+ "message": "Welcome aboard!",
+ },
+ )
+
+
+@router.get("/count", summary="Waitlist count")
+async def waitlist_count():
+ return JSONResponse(
+ status_code=200,
+ content={"data": {"count": len(_load_waitlist())}},
+ )
diff --git a/schemas/translation.py b/schemas/translation.py
index f1094bf..6d81192 100644
--- a/schemas/translation.py
+++ b/schemas/translation.py
@@ -146,6 +146,14 @@ class TranslationStatusData(BaseModel):
description="Message d'erreur si status='failed'",
example=None
)
+ quality: Optional[dict] = Field(
+ None,
+ description=(
+ "Rapport qualité post-traduction (job réussi) : "
+ "score 0-100, fidélité des nombres, ratio non traduit."
+ ),
+ example={"score": 92, "numbers": {"source_numbers": 12, "preserved": 12, "fidelity": 1.0}, "untranslated_ratio": 0.02},
+ )
class Config:
json_schema_extra = {
diff --git a/scripts/diagnostic_templates.py b/scripts/diagnostic_templates.py
new file mode 100644
index 0000000..a97dc71
--- /dev/null
+++ b/scripts/diagnostic_templates.py
@@ -0,0 +1,27 @@
+import json
+import os
+from pathlib import Path
+
+GLOSSARIES_DIR = Path(os.getenv("GLOSSARIES_DIR", "data/glossaries"))
+TARGET_LANGUAGES = ["de", "es", "it", "pt", "nl", "ru", "ja", "ko", "zh", "ar", "fa"]
+
+if not GLOSSARIES_DIR.exists():
+ raise SystemExit(f"Glossaries directory not found: {GLOSSARIES_DIR} (set GLOSSARIES_DIR)")
+
+for f in GLOSSARIES_DIR.glob("*.json"):
+ if f.name == "index.json":
+ continue
+ try:
+ with open(f, "r", encoding="utf-8") as file:
+ data = json.load(file)
+ terms = data.get("terms", [])
+ missing_count = 0
+ total_count = len(terms)
+ for t in terms:
+ trans = t.get("translations", {})
+ for lang in TARGET_LANGUAGES:
+ if lang not in trans or not trans[lang]:
+ missing_count += 1
+ print(f"{f.name}: {total_count} termes, {missing_count} traductions manquantes.")
+ except Exception as e:
+ print(f"Error {f.name}: {e}")
diff --git a/services/glossary_service.py b/services/glossary_service.py
index da33b05..00b581a 100644
--- a/services/glossary_service.py
+++ b/services/glossary_service.py
@@ -198,9 +198,11 @@ def build_full_prompt(
source_lang: str = "fr",
target_lang: str = "en",
glossary_target_lang: str = "multi",
+ formality: Optional[str] = None,
) -> str:
"""
- Build the complete prompt combining custom prompt and glossary.
+ Build the complete prompt combining custom prompt, glossary, formality
+ and regional variant directives.
Args:
custom_prompt: Optional custom system prompt from user
@@ -208,6 +210,8 @@ def build_full_prompt(
source_lang: ISO code of the source language
target_lang: ISO code of the target language
glossary_target_lang: ISO code of the glossary's target language configuration
+ formality: Optional tone override — "formal" or "informal". Only
+ meaningful for LLM engines (ignored by classic engines).
Returns:
Combined prompt string
@@ -224,4 +228,30 @@ def build_full_prompt(
if glossary_prompt:
parts.append(glossary_prompt)
+ if formality in ("formal", "informal"):
+ if formality == "formal":
+ parts.append(
+ "TONE: Use a formal, professional register throughout "
+ "(formal address (vous/Sie) where the language distinguishes; "
+ "no slang, no contractions where avoidable)."
+ )
+ else:
+ parts.append(
+ "TONE: Use an informal, natural register throughout "
+ "(tu-style address where the language distinguishes; "
+ "contractions welcome)."
+ )
+
+ # Regional variant: when the target code carries a region (pt-BR,
+ # fr-CA, zh-CN...), make the expected variety explicit — LLMs default
+ # to the dominant variant otherwise (pt-PT, fr-FR...).
+ if target_lang and "-" in target_lang and target_lang != "auto":
+ from core.languages import language_name
+
+ name = language_name(target_lang)
+ if name and name != target_lang:
+ parts.append(
+ f"REGIONAL VARIANT: write specifically in {name}."
+ )
+
return "\n\n".join(parts) if parts else ""
\ No newline at end of file
diff --git a/services/mistral_ocr.py b/services/mistral_ocr.py
new file mode 100644
index 0000000..52b7bdb
--- /dev/null
+++ b/services/mistral_ocr.py
@@ -0,0 +1,216 @@
+"""
+Mistral OCR client — text extraction for scanned PDFs.
+
+Image-only PDFs have no extractable text layer: PyMuPDF sees empty pages
+and the layout-preserving pipeline would output an empty document. This
+client calls the Mistral OCR API to recover the text.
+
+API reference (2026): POST https://api.mistral.ai/v1/ocr with Bearer auth,
+body {"model", "document": {"type": "document_url", "document_url":
+"data:application/pdf;base64,..."}, "pages": [0, 1, ...]}. The response
+contains {"pages": [{"index", "markdown", "dimensions"}, ...]}.
+
+The document is sent in chunks of PAGES_PER_REQUEST pages: it bounds the
+request payload and lets us report progress page by page.
+"""
+
+import base64
+import time
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+import requests
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+MISTRAL_OCR_URL = "https://api.mistral.ai/v1/ocr"
+DEFAULT_MODEL = "mistral-ocr-latest"
+PAGES_PER_REQUEST = 8
+
+MISTRAL_INVALID_KEY = "MISTRAL_INVALID_KEY"
+MISTRAL_QUOTA_EXCEEDED = "MISTRAL_QUOTA_EXCEEDED"
+MISTRAL_TIMEOUT = "MISTRAL_TIMEOUT"
+MISTRAL_SERVICE_ERROR = "MISTRAL_SERVICE_ERROR"
+
+
+class MistralOCRError(Exception):
+ """Raised when the Mistral OCR API cannot extract the PDF text."""
+
+ def __init__(
+ self, code: str, message: str, details: Optional[Dict[str, Any]] = None
+ ):
+ self.code = code
+ self.message = message
+ self.details = details or {}
+ super().__init__(self.message)
+
+
+class MistralOCRClient:
+ """Thin synchronous client around the Mistral OCR endpoint."""
+
+ def __init__(
+ self,
+ api_key: str,
+ model: str = DEFAULT_MODEL,
+ timeout: int = 180,
+ max_retries: int = 2,
+ retry_delay: float = 2.0,
+ ):
+ self._api_key = (api_key or "").strip()
+ self._model = model or DEFAULT_MODEL
+ self._timeout = timeout
+ self._max_retries = max_retries
+ self._retry_delay = retry_delay
+
+ def is_available(self) -> bool:
+ """True when an API key is configured."""
+ return bool(self._api_key)
+
+ def _post_ocr(self, data_uri: str, pages: List[int]) -> List[Dict[str, Any]]:
+ """POST one OCR request for the given 0-based page list, with retries."""
+ payload = {
+ "model": self._model,
+ "document": {"type": "document_url", "document_url": data_uri},
+ "pages": pages,
+ }
+ headers = {
+ "Authorization": f"Bearer {self._api_key}",
+ "Content-Type": "application/json",
+ }
+
+ last_error: Optional[Exception] = None
+ for attempt in range(self._max_retries + 1):
+ try:
+ response = requests.post(
+ MISTRAL_OCR_URL,
+ json=payload,
+ headers=headers,
+ timeout=self._timeout,
+ )
+
+ if response.status_code == 401:
+ raise MistralOCRError(
+ MISTRAL_INVALID_KEY,
+ "Clé API Mistral invalide (MISTRAL_API_KEY).",
+ {"status_code": 401},
+ )
+ if response.status_code in (402, 429):
+ raise MistralOCRError(
+ MISTRAL_QUOTA_EXCEEDED,
+ "Quota Mistral OCR épuisé ou limite de débit atteinte.",
+ {"status_code": response.status_code},
+ )
+ if response.status_code >= 500:
+ raise MistralOCRError(
+ MISTRAL_SERVICE_ERROR,
+ f"Service Mistral OCR indisponible (HTTP {response.status_code}).",
+ {"status_code": response.status_code},
+ )
+ if response.status_code != 200:
+ raise MistralOCRError(
+ MISTRAL_SERVICE_ERROR,
+ f"Erreur Mistral OCR (HTTP {response.status_code}): {response.text[:200]}",
+ {"status_code": response.status_code},
+ )
+
+ pages_out = response.json().get("pages", [])
+ if not pages_out:
+ raise MistralOCRError(
+ MISTRAL_SERVICE_ERROR,
+ "Réponse Mistral OCR vide.",
+ )
+ return pages_out
+
+ except MistralOCRError as e:
+ if e.code in (MISTRAL_INVALID_KEY, MISTRAL_QUOTA_EXCEEDED):
+ raise # not transient
+ last_error = e
+ except requests.exceptions.Timeout as e:
+ last_error = MistralOCRError(
+ MISTRAL_TIMEOUT,
+ f"Délai d'attente Mistral OCR dépassé ({self._timeout}s).",
+ )
+ except requests.exceptions.RequestException as e:
+ last_error = MistralOCRError(
+ MISTRAL_SERVICE_ERROR,
+ f"Mistral OCR injoignable: {str(e)[:150]}",
+ )
+
+ if attempt < self._max_retries:
+ delay = self._retry_delay * (2**attempt)
+ logger.info(
+ "mistral_ocr_retry",
+ attempt=attempt + 1,
+ delay_s=round(delay, 2),
+ error=last_error.code if last_error else "unknown",
+ )
+ time.sleep(delay)
+
+ raise last_error or MistralOCRError(
+ MISTRAL_SERVICE_ERROR, "Erreur Mistral OCR inconnue."
+ )
+
+ def extract_pdf_text(
+ self,
+ pdf_path: Path,
+ progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+ ) -> List[str]:
+ """OCR a PDF file and return one text (markdown) string per page.
+
+ Pages are processed in chunks of ``PAGES_PER_REQUEST``; the returned
+ list is ordered by page index, empty strings for pages OCR returned
+ nothing for.
+ """
+ import fitz
+
+ pdf_path = Path(pdf_path)
+ data_b64 = base64.b64encode(pdf_path.read_bytes()).decode("ascii")
+ data_uri = f"data:application/pdf;base64,{data_b64}"
+
+ with fitz.open(str(pdf_path)) as doc:
+ total_pages = len(doc)
+ if total_pages == 0:
+ raise MistralOCRError(MISTRAL_SERVICE_ERROR, "PDF vide (0 page).")
+
+ chunks = [
+ list(range(start, min(start + PAGES_PER_REQUEST, total_pages)))
+ for start in range(0, total_pages, PAGES_PER_REQUEST)
+ ]
+
+ page_texts: List[str] = [""] * total_pages
+ done_chunks = 0
+ for chunk in chunks:
+ pages_out = self._post_ocr(data_uri, chunk)
+ for page in pages_out:
+ idx = int(page.get("index", -1))
+ if 0 <= idx < total_pages:
+ page_texts[idx] = page.get("markdown", "") or ""
+
+ done_chunks += 1
+ logger.info(
+ "mistral_ocr_chunk_done",
+ pages_done=min(done_chunks * PAGES_PER_REQUEST, total_pages),
+ total_pages=total_pages,
+ )
+ if progress_callback and chunks:
+ pct = int(5 + 20 * done_chunks / len(chunks))
+ progress_callback(
+ {
+ "current": done_chunks,
+ "total": len(chunks),
+ "phase": f"OCR (Mistral) {min(done_chunks * PAGES_PER_REQUEST, total_pages)}/{total_pages}",
+ "paragraph": done_chunks,
+ "total_paragraphs": len(chunks),
+ "progress_override": pct,
+ }
+ )
+
+ extracted = sum(1 for t in page_texts if t.strip())
+ logger.info(
+ "mistral_ocr_extracted",
+ pages_with_text=extracted,
+ total_pages=total_pages,
+ )
+ return page_texts
diff --git a/services/providers/config.py b/services/providers/config.py
index 327113e..30d31c1 100644
--- a/services/providers/config.py
+++ b/services/providers/config.py
@@ -96,38 +96,47 @@ class ProvidersConfig:
DEEPSEEK_MAX_RETRIES: int = int(os.getenv("DEEPSEEK_MAX_RETRIES", "3"))
DEEPSEEK_RETRY_DELAY: float = float(os.getenv("DEEPSEEK_RETRY_DELAY", "1.0"))
- # Minimax (direct API - m2.7, MiniMax-M1)
+ # Minimax (public OpenAI-compatible API - https://api.minimax.io)
MINIMAX_ENABLED: bool = os.getenv("MINIMAX_ENABLED", "false").lower() == "true"
MINIMAX_API_KEY: str = os.getenv("MINIMAX_API_KEY", "")
- MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M1")
- MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.chat/v1")
+ MINIMAX_MODEL: str = os.getenv("MINIMAX_MODEL", "MiniMax-M3")
+ MINIMAX_BASE_URL: str = os.getenv("MINIMAX_BASE_URL", "https://api.minimax.io/v1")
MINIMAX_GROUP_ID: str = os.getenv("MINIMAX_GROUP_ID", "")
MINIMAX_TIMEOUT: int = int(os.getenv("MINIMAX_TIMEOUT", "60"))
MINIMAX_MAX_RETRIES: int = int(os.getenv("MINIMAX_MAX_RETRIES", "3"))
MINIMAX_RETRY_DELAY: float = float(os.getenv("MINIMAX_RETRY_DELAY", "1.0"))
# Fallback chain configuration
- # General fallback chain (backward compatibility)
+ #
+ # IMPORTANT: the registry-based fallback (translate_with_fallback) only
+ # ever sees providers that _auto_register_providers() registers, i.e.
+ # google, deepl, openai, deepseek and minimax. The OpenAI-compatible
+ # shims (openrouter, openrouter_premium, zai) and google_cloud are wired
+ # directly in routes/translate_routes.py and are intentionally NOT part of
+ # the registry fallback chain — listing them here would make
+ # translate_with_fallback silently skip them with a "provider not
+ # registered" log on every call. Override via env if you know what you
+ # are doing.
FALLBACK_CHAIN: List[str] = [
name.strip()
for name in os.getenv(
- "PROVIDER_FALLBACK_CHAIN", "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
+ "PROVIDER_FALLBACK_CHAIN", "google,deepl,openai,deepseek,minimax"
).split(",")
if name.strip()
]
# Mode-specific fallback chains
- # Classic mode: Google Translate -> Google Cloud -> DeepL
+ # Classic mode: Google Translate -> DeepL
FALLBACK_CHAIN_CLASSIC: List[str] = [
name.strip()
- for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,google_cloud,deepl").split(",")
+ for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,deepl").split(",")
if name.strip()
]
- # LLM mode: cloud providers in order of cost/quality (no Ollama by default)
+ # LLM mode: cloud providers in order of cost/quality (registry-registered only)
FALLBACK_CHAIN_LLM: List[str] = [
name.strip()
- for name in os.getenv("FALLBACK_CHAIN_LLM", "openrouter,openrouter_premium,openai,deepseek,zai").split(",")
+ for name in os.getenv("FALLBACK_CHAIN_LLM", "openai,deepseek,minimax").split(",")
if name.strip()
]
diff --git a/services/providers/deepseek_provider.py b/services/providers/deepseek_provider.py
index b73f5af..1f13554 100644
--- a/services/providers/deepseek_provider.py
+++ b/services/providers/deepseek_provider.py
@@ -39,16 +39,10 @@ Rules:
def _get_language_name(code: str) -> str:
- language_names = {
- "en": "English", "fr": "French", "es": "Spanish", "de": "German",
- "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
- "zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
- "hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
- "th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
- "ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
- }
- return language_names.get(code.split("-")[0].lower(), code)
+ """Convert language code to full name (all supported languages)."""
+ from core.languages import language_name
+ return language_name(code)
class DeepSeekProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -161,9 +155,19 @@ class DeepSeekTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
- system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
+ _base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
+ # Base translation instructions always present; the custom prompt
+ # (glossary/tone/context) is appended, never a replacement.
+ if custom_prompt and custom_prompt.strip():
+ system_prompt = (
+ _base_prompt
+ + "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ + custom_prompt.strip()
+ )
+ else:
+ system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):
diff --git a/services/providers/fallback.py b/services/providers/fallback.py
index 7481ce3..0b2c6fb 100644
--- a/services/providers/fallback.py
+++ b/services/providers/fallback.py
@@ -17,7 +17,15 @@ import time
from core.logging import get_logger
logger = get_logger(__name__)
-_HAS_STRUCTLOG = True
+
+# Detect structlog rather than hardcoding True, so the stdlib-logging fallback
+# branches below stay reachable if structlog is ever absent.
+try:
+ import structlog # noqa: F401
+
+ _HAS_STRUCTLOG = True
+except ImportError: # pragma: no cover - structlog is a hard project dep
+ _HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
diff --git a/services/providers/minimax_provider.py b/services/providers/minimax_provider.py
index 495e1bc..7e36e98 100644
--- a/services/providers/minimax_provider.py
+++ b/services/providers/minimax_provider.py
@@ -1,7 +1,9 @@
"""
-Minimax Provider - Cloud LLM translation via Minimax API (m2.7).
+Minimax Provider - Cloud LLM translation via the Minimax public API.
-Minimax uses an OpenAI-compatible Chat Completions API.
+Minimax exposes an OpenAI-compatible Chat Completions API at
+``https://api.minimax.io/v1/chat/completions`` (default model ``MiniMax-M3``).
+Note: ``api.minimax.chat`` is NOT a reachable public host.
"""
import threading
@@ -39,16 +41,10 @@ Rules:
def _get_language_name(code: str) -> str:
- language_names = {
- "en": "English", "fr": "French", "es": "Spanish", "de": "German",
- "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian",
- "zh": "Chinese", "ja": "Japanese", "ko": "Korean", "ar": "Arabic",
- "hi": "Hindi", "tr": "Turkish", "pl": "Polish", "vi": "Vietnamese",
- "th": "Thai", "uk": "Ukrainian", "cs": "Czech", "sv": "Swedish",
- "ro": "Romanian", "hu": "Hungarian", "el": "Greek", "he": "Hebrew",
- }
- return language_names.get(code.split("-")[0].lower(), code)
+ """Convert language code to full name (all supported languages)."""
+ from core.languages import language_name
+ return language_name(code)
class MinimaxProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
@@ -62,17 +58,19 @@ class MinimaxTranslationProvider(TranslationProvider):
"""
Minimax translation provider using OpenAI-compatible API.
- Default model: MiniMax-M1 (latest). Also supports m2.7 via env config.
+ Default model: MiniMax-M3 (latest public OpenAI-compatible model).
+ The public endpoint is https://api.minimax.io/v1 (NOT api.minimax.chat,
+ which is not a reachable host on the public API).
"""
def __init__(
self,
api_key: str,
- model: str = "MiniMax-M1",
+ model: str = "MiniMax-M3",
timeout: int = 60,
max_retries: int = 3,
retry_delay: float = 1.0,
- base_url: str = "https://api.minimax.chat/v1",
+ base_url: str = "https://api.minimax.io/v1",
group_id: str = "",
):
if not api_key or not api_key.strip():
@@ -144,13 +142,24 @@ class MinimaxTranslationProvider(TranslationProvider):
def get_name(self) -> str:
return self._provider_name
- def is_available(self) -> bool:
+ def _probe_available(self) -> tuple[bool, int]:
+ """Probe the Minimax API. Returns (available, status_code).
+
+ Minimax does not document a public ``GET /models`` endpoint, so a 404/405
+ on that path does NOT mean the provider is down — it only means the path
+ is absent. We treat any non-401 response as "available": the host is
+ reachable and the API key was not rejected. Only 401 (and network
+ errors) mark the provider unavailable.
+ """
try:
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
- return response.status_code == 200
+ return response.status_code != 401, response.status_code
except Exception:
- return False
+ return False, 0
+
+ def is_available(self) -> bool:
+ return self._probe_available()[0]
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
text = request.text
@@ -163,9 +172,19 @@ class MinimaxTranslationProvider(TranslationProvider):
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = request.metadata.get("custom_prompt") if request.metadata else None
- system_prompt = custom_prompt or DEFAULT_TRANSLATION_PROMPT.format(
+ _base_prompt = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang_name, target_lang=target_lang_name
)
+ # Base translation instructions always present; the custom prompt
+ # (glossary/tone/context) is appended, never a replacement.
+ if custom_prompt and custom_prompt.strip():
+ system_prompt = (
+ _base_prompt
+ + "\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n"
+ + custom_prompt.strip()
+ )
+ else:
+ system_prompt = _base_prompt
last_error = None
for attempt in range(self._max_retries + 1):
@@ -199,20 +218,19 @@ class MinimaxTranslationProvider(TranslationProvider):
def health_check(self) -> ProviderHealthStatus:
start_time = time.time()
- try:
- headers = {"Authorization": f"Bearer {self._api_key}"}
- response = requests.get(f"{self._base_url}/models", headers=headers, timeout=5)
- latency_ms = (time.time() - start_time) * 1000
+ available, status_code = self._probe_available()
+ latency_ms = (time.time() - start_time) * 1000
+ if available:
return ProviderHealthStatus(
- name=self._provider_name, available=response.status_code == 200,
+ name=self._provider_name, available=True,
latency_ms=round(latency_ms, 2), last_check=datetime.now(timezone.utc).isoformat(),
model=self._model)
- except Exception as e:
- return ProviderHealthStatus(
- name=self._provider_name, available=False,
- latency_ms=round((time.time() - start_time) * 1000, 2),
- error=str(e)[:100], last_check=datetime.now(timezone.utc).isoformat(),
- model=self._model)
+ return ProviderHealthStatus(
+ name=self._provider_name, available=False,
+ latency_ms=round(latency_ms, 2),
+ error=f"probe failed (status={status_code})"[:100],
+ last_check=datetime.now(timezone.utc).isoformat(),
+ model=self._model)
_provider_instance: Optional[MinimaxTranslationProvider] = None
diff --git a/services/providers/openai_provider.py b/services/providers/openai_provider.py
index 02cd9e1..f812064 100644
--- a/services/providers/openai_provider.py
+++ b/services/providers/openai_provider.py
@@ -22,7 +22,16 @@ from typing import Any, Dict, List, Optional
from core.logging import get_logger
logger = get_logger(__name__)
-_HAS_STRUCTLOG = True
+
+# structlog is the project's logger backend (see core/logging.py), but detect
+# it rather than hardcoding True so the stdlib-logging fallback branches below
+# are reachable if structlog is ever absent.
+try:
+ import structlog # noqa: F401
+
+ _HAS_STRUCTLOG = True
+except ImportError: # pragma: no cover - structlog is a hard project dep
+ _HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
@@ -111,56 +120,26 @@ Rules:
def _build_system_prompt(
source_lang: str, target_lang: str, custom_prompt: Optional[str] = None
) -> str:
- """Build system prompt for translation."""
- if custom_prompt:
- return custom_prompt
- return DEFAULT_TRANSLATION_PROMPT.format(
+ """Build system prompt for translation.
+
+ The base translation instructions are ALWAYS present — a custom prompt
+ (glossary, tone, context) is appended as additional directives, never a
+ replacement. Previously a glossary-only custom prompt produced a system
+ prompt with no translation instruction at all.
+ """
+ base = DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang, target_lang=target_lang
)
+ if custom_prompt and custom_prompt.strip():
+ return f"{base}\n\nADDITIONAL CONTEXT AND INSTRUCTIONS:\n{custom_prompt.strip()}"
+ return base
def _get_language_name(code: str) -> str:
"""Convert language code to full name for better LLM understanding."""
- language_names = {
- "en": "English",
- "fr": "French",
- "es": "Spanish",
- "de": "German",
- "it": "Italian",
- "pt": "Portuguese",
- "nl": "Dutch",
- "ru": "Russian",
- "zh": "Chinese",
- "ja": "Japanese",
- "ko": "Korean",
- "ar": "Arabic",
- "hi": "Hindi",
- "tr": "Turkish",
- "pl": "Polish",
- "vi": "Vietnamese",
- "th": "Thai",
- "id": "Indonesian",
- "ms": "Malay",
- "uk": "Ukrainian",
- "cs": "Czech",
- "sv": "Swedish",
- "da": "Danish",
- "fi": "Finnish",
- "no": "Norwegian",
- "el": "Greek",
- "he": "Hebrew",
- "ro": "Romanian",
- "hu": "Hungarian",
- "bg": "Bulgarian",
- "sk": "Slovak",
- "hr": "Croatian",
- "sl": "Slovenian",
- "lt": "Lithuanian",
- "lv": "Latvian",
- "et": "Estonian",
- }
- base_code = code.split("-")[0].lower()
- return language_names.get(base_code, code)
+ from core.languages import language_name
+
+ return language_name(code)
class OpenAITranslationProvider(TranslationProvider):
@@ -529,21 +508,120 @@ class OpenAITranslationProvider(TranslationProvider):
error_code=OPENAI_SERVICE_ERROR,
)
+ def _make_batch_api_request(
+ self, texts: List[str], system_prompt: str
+ ) -> Optional[List[str]]:
+ """Translate a whole chunk in ONE request via a numbered JSON list.
+
+ The user message is a JSON array; the model must answer with a JSON
+ array of the same length. Returns None when the answer cannot be
+ parsed confidently — callers then fall back to per-item calls
+ (correctness over latency).
+ """
+ import json as _json
+
+ numbered = _json.dumps(
+ [{"id": i, "text": t} for i, t in enumerate(texts)],
+ ensure_ascii=False,
+ )
+ batch_system = (
+ system_prompt
+ + "\n\nBATCH MODE: the user message is a JSON array of items with "
+ "unique ids. Answer with ONLY a JSON array of objects "
+ '[{"id": , "translation": ""}], same '
+ "length and same ids, in the same order. Translate every item; "
+ "keep ids unchanged; no comments, no markdown fence."
+ )
+
+ try:
+ content, _usage = self._make_api_request(numbered, batch_system)
+ raw = content.strip()
+ # Strip an optional markdown fence
+ if raw.startswith("```"):
+ raw = raw.strip("`")
+ if raw.lower().startswith("json"):
+ raw = raw[4:]
+ raw = raw.strip()
+ parsed = _json.loads(raw)
+ if not isinstance(parsed, list) or len(parsed) != len(texts):
+ return None
+ out: List[str] = [""] * len(texts)
+ for item in parsed:
+ if not isinstance(item, dict):
+ return None
+ idx = item.get("id")
+ translation = item.get("translation")
+ if not isinstance(idx, int) or not 0 <= idx < len(texts):
+ return None
+ if not isinstance(translation, str) or not translation.strip():
+ return None
+ out[idx] = translation.strip()
+ return out
+ except OpenAIProviderError:
+ raise
+ except Exception:
+ return None
+
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts.
- Args:
- requests: List of TranslationRequest objects
-
- Returns:
- List of TranslationResponse objects
+ Chunks arrive from the translators as ~15 texts. When every request
+ shares the same language pair and metadata, they are sent in ONE
+ call (numbered JSON list — ~15× fewer requests, better contextual
+ consistency across neighbouring segments). Any parse/API doubt falls
+ back to the per-item path so a batch failure never corrupts output.
"""
if not requests:
return []
+ same_pair = len({(r.source_language, r.target_language) for r in requests}) == 1
+ same_meta = len(
+ {tuple(sorted((r.metadata or {}).items())) for r in requests}
+ ) == 1
+
+ if same_pair and same_meta and len(requests) > 1:
+ try:
+ source_lang_name = _get_language_name(
+ requests[0].source_language or "auto"
+ ) or "the source language (auto-detect)"
+ target_lang_name = _get_language_name(requests[0].target_language)
+ custom_prompt = None
+ if requests[0].metadata:
+ custom_prompt = requests[0].metadata.get("custom_prompt")
+ system_prompt = _build_system_prompt(
+ source_lang_name, target_lang_name, custom_prompt
+ )
+ texts = [r.text for r in requests]
+ translations = self._make_batch_api_request(texts, system_prompt)
+ if translations is not None:
+ _log_info(
+ "openai_batch_translation_success",
+ items=len(requests),
+ model=self._model,
+ )
+ return [
+ TranslationResponse(
+ translated_text=t,
+ provider_name=self._provider_name,
+ from_cache=False,
+ )
+ for t in translations
+ ]
+ _log_warning(
+ "openai_batch_translation_fallback",
+ reason="unparseable_response",
+ items=len(requests),
+ )
+ except Exception as e:
+ _log_warning(
+ "openai_batch_translation_fallback",
+ reason=type(e).__name__,
+ items=len(requests),
+ )
+
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
diff --git a/services/quality/qa_report.py b/services/quality/qa_report.py
new file mode 100644
index 0000000..16b1772
--- /dev/null
+++ b/services/quality/qa_report.py
@@ -0,0 +1,119 @@
+"""
+Post-translation QA report (no external API — pure heuristics).
+
+Answers three user-facing questions about a finished job:
+ - Are numbers preserved? (digit-token multiset source vs translation)
+ - Did everything actually get translated? (untranslated-ratio heuristic)
+ - A 0-100 confidence score combining both.
+
+Never blocks a job: every failure degrades to "skipped".
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+# Tokens that are never counted as "content words" for the untranslated ratio
+_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
+_WORD_RE = re.compile(r"[\w']+", re.UNICODE)
+_NUM_RE = re.compile(r"\d+(?:[.,]\d+)*", re.UNICODE)
+
+# Latin-script vs non-Latin word detection for language confusion heuristics
+_LATIN_RE = re.compile(r"[a-zA-Z]")
+
+
+def _extract_text_pairs(source_path: Path, output_path: Path, file_extension: str):
+ """Extract (source_text, translated_text) full-document strings.
+
+ Reuses the quality layer's file extractor so every format is read the
+ same way the L0 check reads it. Applied to the INPUT file the same
+ extractor yields the SOURCE text (the "translated" field simply holds
+ whatever text lives in the file).
+ """
+ from services.quality.file_extractor import extract_sample
+
+ src_chunks = extract_sample(Path(source_path), file_extension, max_samples=10_000)
+ out_chunks = extract_sample(Path(output_path), file_extension, max_samples=10_000)
+ src = "\n".join(c["translated"] for c in src_chunks)
+ out = "\n".join(c["translated"] for c in out_chunks)
+ return src, out
+
+
+def _number_multiset(text: str) -> List[str]:
+ """Digit tokens with the decimal separator normalized (12,50 == 12.50).
+
+ French/English differ on ',' vs '.'; a real translation keeps the value.
+ """
+ return sorted(n.replace(",", ".") for n in _NUM_RE.findall(text))
+
+
+def _number_fidelity(source: str, translated: str) -> Optional[dict]:
+ """Compare digit tokens: how many source numbers survived (order-insensitive)."""
+ src_nums = _number_multiset(source)
+ if not src_nums:
+ return None
+ out_nums = _number_multiset(translated)
+ # multiset intersection
+ from collections import Counter
+
+ src_count = Counter(src_nums)
+ out_count = Counter(out_nums)
+ kept = sum((src_count & out_count).values())
+ return {
+ "source_numbers": len(src_nums),
+ "preserved": kept,
+ "fidelity": round(kept / len(src_nums), 3),
+ }
+
+
+def _untranslated_ratio(source: str, translated: str) -> Optional[float]:
+ """Heuristic: share of source content-words still present verbatim in
+ the output. ~0 for a real translation, ~1 when nothing was translated.
+ """
+ src_words = [w.lower() for w in _WORD_RE.findall(source) if len(w) > 3]
+ if len(src_words) < 10:
+ return None
+ out_lower = translated.lower()
+ hits = sum(1 for w in set(src_words) if w in out_lower)
+ return round(hits / len(set(src_words)), 3)
+
+
+def run_qa_report(
+ source_path: Path, output_path: Path, target_lang: str, file_extension: str
+) -> Optional[Dict]:
+ """Compute the QA report for a finished translation job.
+
+ Returns a dict with numbers fidelity, untranslated ratio and a
+ 0-100 score, or None if the report could not be computed.
+ """
+ try:
+ source, translated = _extract_text_pairs(
+ Path(source_path), Path(output_path), file_extension
+ )
+ except Exception as e:
+ logger.warning("qa_report_extract_failed", error=str(e))
+ return None
+
+ if not source.strip() or not translated.strip():
+ return None
+
+ numbers = _number_fidelity(source, translated)
+ untranslated = _untranslated_ratio(source, translated)
+
+ score = 100.0
+ if numbers:
+ score *= 0.5 + 0.5 * numbers["fidelity"]
+ if untranslated is not None and untranslated > 0:
+ score *= max(0.0, 1.0 - untranslated)
+
+ report = {
+ "score": int(round(score)),
+ "numbers": numbers,
+ "untranslated_ratio": untranslated,
+ }
+ logger.info("qa_report_computed", **{k: v for k, v in report.items() if v is not None})
+ return report
diff --git a/services/translation_service.py b/services/translation_service.py
index d861ab6..f2de749 100644
--- a/services/translation_service.py
+++ b/services/translation_service.py
@@ -23,23 +23,11 @@ from core.logging import get_logger
logger = get_logger(__name__)
-# Map language codes to full names for LLM prompts (models understand "French" better than "fr")
-_LLM_LANG_NAMES = {
- "en": "English", "es": "Spanish", "de": "German", "fr": "French", "ja": "Japanese",
- "pt": "Portuguese", "ru": "Russian", "it": "Italian", "zh": "Chinese", "zh-CN": "Chinese (Simplified)",
- "zh-TW": "Chinese (Traditional)", "pl": "Polish", "nl": "Dutch", "tr": "Turkish", "ko": "Korean",
- "ar": "Arabic", "fa": "Persian", "vi": "Vietnamese", "id": "Indonesian", "uk": "Ukrainian",
- "sv": "Swedish", "cs": "Czech", "el": "Greek", "he": "Hebrew", "hi": "Hindi", "ro": "Romanian",
- "da": "Danish", "fi": "Finnish", "no": "Norwegian", "hu": "Hungarian", "th": "Thai",
- "sk": "Slovak", "bg": "Bulgarian", "hr": "Croatian", "ca": "Catalan", "ms": "Malay",
-}
-
-
def _lang_name(code: str) -> str:
"""Return full language name for LLM prompts; fallback to code if unknown."""
- if not code or code == "auto":
- return ""
- return _LLM_LANG_NAMES.get(code, _LLM_LANG_NAMES.get(code.split("-")[0], code))
+ from core.languages import language_name
+
+ return language_name(code)
# Global thread pool for parallel translations
@@ -1191,13 +1179,16 @@ class TranslationService:
if not self.translate_images:
return ""
- # Ollama, OpenAI, and OpenRouter support image translation
- if isinstance(self.provider, OllamaTranslationProvider):
- return self.provider.translate_image(image_path, target_language)
- elif isinstance(self.provider, OpenAITranslationProvider):
- return self.provider.translate_image(image_path, target_language)
- elif isinstance(self.provider, OpenRouterTranslationProvider):
- return self.provider.translate_image(image_path, target_language)
+ # Duck-typing: any provider that exposes ``translate_image`` can be
+ # used, regardless of whether it is a new-style (services/providers/*)
+ # or legacy (services/translation_service) instance. The previous
+ # isinstance() checks only matched the legacy classes, so a new-style
+ # provider wired in by the route never reached this branch.
+ if hasattr(self.provider, "translate_image"):
+ try:
+ return self.provider.translate_image(image_path, target_language)
+ except Exception:
+ return ""
return ""
diff --git a/services/translation_tm.py b/services/translation_tm.py
new file mode 100644
index 0000000..179b1d3
--- /dev/null
+++ b/services/translation_tm.py
@@ -0,0 +1,199 @@
+"""
+Translation Memory (TM) — persistent reuse layer on top of the existing
+Redis-backed cache (services/translation_cache.py).
+
+Scope is PER USER (privacy: a customer's translations are never served to
+another customer) and per translation context (custom prompt / glossary /
+formality are hashed into the key so a glossary change invalidates old
+matches). When Redis is not configured the cache transparently falls back
+to a process-local LRU — still useful within a worker's lifetime.
+"""
+
+import hashlib
+from typing import Dict, List, Optional, Tuple
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class TMScope:
+ """Identity/context of a translation job for TM keying."""
+
+ __slots__ = ("user_id", "context_hash")
+
+ def __init__(self, user_id: Optional[str], context_hash: Optional[str]):
+ self.user_id = user_id
+ self.context_hash = context_hash
+
+ @classmethod
+ def from_prompt(cls, user_id: Optional[str], prompt: Optional[str]) -> "TMScope":
+ """Build a scope; the prompt (glossary + tone + formality merged)
+ is hashed so any directive change produces different TM entries."""
+ context_hash = None
+ if prompt:
+ context_hash = hashlib.sha256(prompt.encode("utf-8")).hexdigest()[:16]
+ return cls(user_id=user_id, context_hash=context_hash)
+
+
+def _cache():
+ from services.translation_cache import get_cache
+
+ return get_cache()
+
+
+def lookup_tm(
+ texts: List[str],
+ target_language: str,
+ source_language: str,
+ provider_name: str,
+ scope: Optional[TMScope],
+) -> Tuple[Dict[int, str], List[int]]:
+ """Return ({index: cached_translation}, [indices not in TM]).
+
+ No-op (all misses) when the scope has no user_id — anonymous jobs are
+ never cached or served from another user's entries.
+ """
+ if not scope or not scope.user_id:
+ return {}, list(range(len(texts)))
+
+ try:
+ cache = _cache()
+ except Exception as e:
+ logger.warning("tm_init_failed", error=str(e))
+ return {}, list(range(len(texts)))
+
+ hits: Dict[int, str] = {}
+ misses: List[int] = []
+ for i, text in enumerate(texts):
+ if not text or not text.strip():
+ misses.append(i)
+ continue
+ cached = cache.get(
+ text,
+ target_language,
+ source_language,
+ provider_name,
+ user_id=scope.user_id,
+ custom_prompt_hash=scope.context_hash,
+ )
+ if cached is not None and cached.strip():
+ hits[i] = cached
+ else:
+ misses.append(i)
+
+ if hits:
+ logger.info("tm_lookup_hits", hits=len(hits), misses=len(misses))
+ return hits, misses
+
+
+def store_tm(
+ texts: List[str],
+ translations: List[str],
+ target_language: str,
+ source_language: str,
+ provider_name: str,
+ scope: Optional[TMScope],
+) -> int:
+ """Store fresh translations in the TM. Returns the number stored."""
+ if not scope or not scope.user_id:
+ return 0
+ try:
+ cache = _cache()
+ except Exception as e:
+ logger.warning("tm_init_failed", error=str(e))
+ return 0
+
+ stored = 0
+ for text, translation in zip(texts, translations):
+ if not text or not translation:
+ continue
+ # Never store identity entries — they would poison future lookups
+ # (an unchanged text is not a translation).
+ if translation.strip() == text.strip():
+ continue
+ cache.set(
+ text,
+ target_language,
+ source_language,
+ provider_name,
+ translation,
+ user_id=scope.user_id,
+ custom_prompt_hash=scope.context_hash,
+ )
+ stored += 1
+ if stored:
+ logger.info("tm_stored", entries=stored)
+ return stored
+
+
+def tm_stats() -> Dict:
+ """Backend stats for observability."""
+ try:
+ return _cache().stats()
+ except Exception:
+ return {}
+
+
+def translate_with_tm(
+ texts: List[str],
+ target_language: str,
+ source_language: str,
+ provider_name: str,
+ scope: "TMScope | None",
+ translate_fn,
+) -> List[str]:
+ """Translate a batch with TM reuse: hits come from the cache, misses go
+ through ``translate_fn`` (which receives ONLY the missed texts, in
+ order) and fresh results are stored back. Falls back to a plain
+ ``translate_fn(texts)`` call when the TM is unavailable.
+ """
+ if not texts:
+ return []
+
+ tm_hits, miss_indices = lookup_tm(
+ texts, target_language, source_language, provider_name, scope
+ )
+ miss_texts = [texts[i] for i in miss_indices]
+
+ if not miss_texts:
+ return [tm_hits.get(i, texts[i]) for i in range(len(texts))]
+
+ try:
+ translated_misses = translate_fn(miss_texts)
+ except Exception:
+ if tm_hits:
+ # Provider failed but we have partial TM hits — keep them and
+ # leave the misses untouched rather than dropping everything.
+ logger.warning("tm_translate_fn_failed_partial_hits", hits=len(tm_hits))
+ translated_misses = None
+ else:
+ raise
+
+ if translated_misses is None:
+ return [
+ tm_hits.get(i, texts[i]) for i in range(len(texts))
+ ]
+
+ store_tm(
+ miss_texts,
+ translated_misses,
+ target_language,
+ source_language,
+ provider_name,
+ scope,
+ )
+
+ merged: List[str] = []
+ miss_pos = 0
+ for i in range(len(texts)):
+ if i in tm_hits:
+ merged.append(tm_hits[i])
+ else:
+ merged.append(
+ translated_misses[miss_pos]
+ if miss_pos < len(translated_misses)
+ else texts[i]
+ )
+ miss_pos += 1
+ return merged
diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py
index 6841535..a9f043a 100644
--- a/tests/test_cleanup.py
+++ b/tests/test_cleanup.py
@@ -101,7 +101,11 @@ def _get_redis_patcher(mock_redis):
@pytest.mark.asyncio
async def test_orphan_deletion(temp_dirs):
- """Test that orphaned files are deleted as per Story 2.15 (AC: #4)"""
+ """Test that orphaned files are deleted as per Story 2.15 (AC: #4).
+
+ Since the security fix (audit 2026-08-26), an orphan is only deleted
+ after a grace period, so young in-flight files are never removed.
+ """
cleanup_mod = _get_cleanup_module()
FileCleanupManager = cleanup_mod.FileCleanupManager
@@ -117,6 +121,10 @@ async def test_orphan_deletion(temp_dirs):
manager = FileCleanupManager(uploads, outputs, temp, cleanup_interval_minutes=5)
+ # Make the orphan older than the grace period (default 15 min)
+ old = time.time() - (manager.orphan_grace_seconds + 60)
+ os.utime(orphan_file, (old, old))
+
mock_redis = AsyncMock()
mock_redis.keys.return_value = ["translation:file:job1"]
mock_redis.get.return_value = json.dumps(
@@ -182,6 +190,11 @@ async def test_cleanup_resilience(temp_dirs):
f2 = uploads / "file2.txt"
f2.write_text("file2")
+ # Make the files older than the grace period so cleanup actually deletes them
+ old = time.time() - 7200
+ os.utime(f1, (old, old))
+ os.utime(f2, (old, old))
+
manager = FileCleanupManager(uploads, outputs, temp, max_file_age_minutes=1)
original_unlink = Path.unlink
diff --git a/tests/test_download_endpoint.py b/tests/test_download_endpoint.py
index 7eb386b..81d0277 100644
--- a/tests/test_download_endpoint.py
+++ b/tests/test_download_endpoint.py
@@ -17,6 +17,8 @@ DOWNLOAD_URL = "/api/v1/download"
REGISTER_URL = "/api/v1/auth/register"
LOGIN_URL = "/api/v1/auth/login"
+AUTH_USER_ID = None # set by the authenticated_client fixture
+
VALID_USER = {
"email": "download@example.com",
"password": "Password123!",
@@ -135,7 +137,9 @@ def client(users_file: Path, monkeypatch):
@pytest.fixture()
def authenticated_client(client):
"""Client avec un utilisateur enregistre et authentifie."""
- client.post(REGISTER_URL, json=VALID_USER)
+ global AUTH_USER_ID
+ reg = client.post(REGISTER_URL, json=VALID_USER)
+ AUTH_USER_ID = reg.json()["data"]["id"]
response = client.post(
LOGIN_URL,
json={
@@ -184,6 +188,7 @@ class TestDownloadEndpoint:
job_id = "tr_test_no_output"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "test.xlsx",
"output_path": None,
@@ -205,6 +210,7 @@ class TestDownloadEndpoint:
job_id = "tr_deleted_disk"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "deleted.xlsx",
"file_extension": ".xlsx",
@@ -224,6 +230,7 @@ class TestDownloadEndpoint:
job_id = "tr_test_processing"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "processing",
"progress_percent": 50,
"file_name": "test.xlsx",
@@ -241,6 +248,7 @@ class TestDownloadEndpoint:
job_id = "tr_test_queued"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "queued",
"file_name": "test.xlsx",
}
@@ -257,6 +265,7 @@ class TestDownloadEndpoint:
job_id = "tr_test_failed"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "failed",
"error_message": "Something went wrong",
"file_name": "test.xlsx",
@@ -288,6 +297,7 @@ class TestContentDisposition:
job_id = "tr_test_disposition"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "report.xlsx",
"file_extension": ".xlsx",
@@ -310,6 +320,7 @@ class TestContentDisposition:
job_id = "tr_test_docx"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "document.docx",
"file_extension": ".docx",
@@ -331,6 +342,7 @@ class TestContentDisposition:
job_id = "tr_test_pptx"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "presentation.pptx",
"file_extension": ".pptx",
@@ -364,6 +376,7 @@ class TestFileDeletion:
job_id = "tr_test_delete"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "to_delete.xlsx",
"file_extension": ".xlsx",
@@ -400,6 +413,7 @@ class TestMIMETypes:
job_id = "tr_test_mime_xlsx"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "test.xlsx",
"file_extension": ".xlsx",
@@ -424,6 +438,7 @@ class TestMIMETypes:
job_id = "tr_test_mime_docx"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "test.docx",
"file_extension": ".docx",
@@ -448,6 +463,7 @@ class TestMIMETypes:
job_id = "tr_test_mime_pptx"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "test.pptx",
"file_extension": ".pptx",
@@ -489,6 +505,7 @@ class TestFileExpired:
job_id = "tr_test_not_ready_msg"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "processing",
"progress_percent": 30,
"file_name": "test.xlsx",
@@ -520,11 +537,12 @@ class TestDownloadIntegration:
job_id = "tr_test_binary"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "binary_test.xlsx",
"file_extension": ".xlsx",
"output_path": str(output_file),
- "user_id": None,
+ "user_id": AUTH_USER_ID,
}
response = authenticated_client.get(f"{DOWNLOAD_URL}/{job_id}")
@@ -557,6 +575,7 @@ class TestErrorDetails:
job_id = "tr_test_details"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "processing",
"progress_percent": 45,
"file_name": "test.xlsx",
@@ -589,6 +608,7 @@ class TestDownloadAuthorization:
job_id = "tr_other_user123"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "other.xlsx",
"file_extension": ".xlsx",
@@ -622,18 +642,19 @@ class TestDownloadAuthorization:
job_id = "tr_own_file123"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": AUTH_USER_ID,
"status": "completed",
"file_name": "own.xlsx",
"file_extension": ".xlsx",
"output_path": str(output_file),
- "user_id": None,
+ "user_id": AUTH_USER_ID,
}
response = authenticated_client.get(f"{DOWNLOAD_URL}/{job_id}")
assert response.status_code == 200
- def test_anonymous_user_can_download_public_job(self, client, tmp_path):
- """Anonymous users can download jobs without user_id (public)"""
+ def test_anonymous_job_requires_token(self, client, tmp_path):
+ """Jobs without an owner require the secret per-job token (fix 2026-08-26)"""
from routes import translate_routes
output_file = tmp_path / "public_job.xlsx"
@@ -642,12 +663,19 @@ class TestDownloadAuthorization:
job_id = "tr_public_job99"
translate_routes._translation_jobs[job_id] = {
"id": job_id,
+ "user_id": None,
+ "access_token": "secret_job_token",
"status": "completed",
"file_name": "public.xlsx",
"file_extension": ".xlsx",
"output_path": str(output_file),
- "user_id": None,
}
+ # Without the token: denied
response = client.get(f"{DOWNLOAD_URL}/{job_id}")
+ assert response.status_code == 403
+ assert response.json()["error"] == "ACCESS_DENIED"
+
+ # With the correct token: allowed
+ response = client.get(f"{DOWNLOAD_URL}/{job_id}?token=secret_job_token")
assert response.status_code == 200
diff --git a/tests/test_excel_sheet_rename_refs.py b/tests/test_excel_sheet_rename_refs.py
new file mode 100644
index 0000000..da50ac2
--- /dev/null
+++ b/tests/test_excel_sheet_rename_refs.py
@@ -0,0 +1,92 @@
+"""Sheet-name translation must not break references.
+
+openpyxl does not rewrite references on rename — cell formulas, defined
+names and validations pointing at a translated sheet would break (#REF!).
+These tests cover the full rename + rewrite path.
+"""
+
+from openpyxl import Workbook, load_workbook
+from openpyxl.workbook.defined_name import DefinedName
+
+from translators.excel_translator import ExcelTranslator
+
+
+class _FrToEn:
+ """Minimal legacy-style provider: French sheet name → English."""
+
+ def translate_batch(self, texts, target_language, source_language="auto"):
+ mapping = {
+ "Ventes": "Sales",
+ "Données": "Data",
+ "Rapport des ventes": "Sales report",
+ "Total": "Total",
+ }
+ return [mapping.get(t, t) for t in texts]
+
+
+def _make_workbook(path):
+ wb = Workbook()
+ ws = wb.active
+ ws.title = "Ventes"
+ ws["A1"] = "Rapport des ventes"
+ ws["A2"] = "Total"
+ ws["A3"] = 10
+ ws["A4"] = 20
+ # Cross-sheet formula (unquoted name)
+ ws2 = wb.create_sheet("Données")
+ ws2["A1"] = "=SUM(Ventes!A3:A4)"
+ # Quoted name (name would need quotes if it had spaces — keep simple here)
+ ws2["A2"] = "=Ventes!A3+Ventes!A4"
+ # Defined name pointing at the renamed sheet
+ wb.defined_names["TotalVentes"] = DefinedName(
+ "TotalVentes", attr_text="'Ventes'!$A$3"
+ )
+ wb.save(path)
+ return wb
+
+
+class TestSheetRenameReferences:
+ def test_cell_formulas_and_defined_names_rewritten(self, tmp_path):
+ from pathlib import Path
+
+ src = tmp_path / "in.xlsx"
+ out = tmp_path / "out.xlsx"
+ _make_workbook(src)
+
+ translator = ExcelTranslator(provider=_FrToEn())
+ translator.translate_file(Path(src), Path(out), "en", "fr")
+
+ wb = load_workbook(out)
+ data = wb["Data"]
+ assert data["A1"].value == "=SUM(Sales!A3:A4)", data["A1"].value
+ assert data["A2"].value == "=Sales!A3+Sales!A4", data["A2"].value
+
+ # Defined name follows the rename
+ dn = wb.defined_names["TotalVentes"]
+ assert "Sales" in (dn.attr_text or ""), dn.attr_text
+ assert "Ventes" not in (dn.attr_text or "")
+
+ # The renamed sheet actually exists under its new name
+ assert "Sales" in wb.sheetnames
+ assert "Ventes" not in wb.sheetnames
+
+ def test_3d_and_multiple_refs(self, tmp_path):
+ mapping = {"Sheet1": "Feuille1", "Sheet2": "Feuille2"}
+ formula = "=SUM(Sheet1!A1:Sheet2!B2)+Sheet1!C3"
+ out = ExcelTranslator._rewrite_sheet_refs_in_formula(formula, mapping)
+ assert out == "=SUM(Feuille1!A1:Feuille2!B2)+Feuille1!C3"
+
+ def test_quoted_refs_and_prefix_safety(self, tmp_path):
+ mapping = {"Ventes": "Sales", "Ventes 2026": "Sales 2026"}
+ out = ExcelTranslator._rewrite_sheet_refs_in_formula(
+ "=SUM('Ventes 2026'!A1:A2)+'Ventes'!B1", mapping
+ )
+ # "Ventes 2026" (longest first) keeps its quotes (name with space);
+ # "Sales" needs no quotes so the canonical unquoted form is emitted.
+ assert out == "=SUM('Sales 2026'!A1:A2)+Sales!B1"
+
+ def test_non_sheet_bang_not_touched(self, tmp_path):
+ mapping = {"Ventes": "Sales"}
+ # "Total!A1" is not a renamed sheet — must stay untouched
+ out = ExcelTranslator._rewrite_sheet_refs_in_formula("=Total!A1", mapping)
+ assert out == "=Total!A1"
diff --git a/tests/test_language_validation.py b/tests/test_language_validation.py
new file mode 100644
index 0000000..c59aa03
--- /dev/null
+++ b/tests/test_language_validation.py
@@ -0,0 +1,45 @@
+"""LanguageValidator — case-insensitive codes and canonical form (zh-CN fix)."""
+
+import pytest
+
+from middleware.validation import LanguageValidator, ValidationError
+
+
+class TestLanguageValidatorCaseInsensitive:
+ def test_zh_cn_mixed_case_accepted(self):
+ assert LanguageValidator.validate("zh-CN") == "zh-CN"
+
+ def test_zh_cn_lowercase_normalized(self):
+ assert LanguageValidator.validate("zh-cn") == "zh-CN"
+
+ def test_zh_cn_uppercase_normalized(self):
+ assert LanguageValidator.validate("ZH-CN") == "zh-CN"
+
+ def test_zh_tw_accepted(self):
+ assert LanguageValidator.validate("zh-TW") == "zh-TW"
+ assert LanguageValidator.validate("zh-tw") == "zh-TW"
+
+ def test_alias_chinese(self):
+ assert LanguageValidator.validate("chinese") == "zh-CN"
+
+ def test_alias_tw(self):
+ assert LanguageValidator.validate("tw") == "zh-TW"
+
+ def test_plain_codes_unchanged(self):
+ assert LanguageValidator.validate("en") == "en"
+ assert LanguageValidator.validate("fr") == "fr"
+
+ def test_auto_accepted(self):
+ assert LanguageValidator.validate("auto") == "auto"
+
+ def test_unknown_code_rejected(self):
+ with pytest.raises(ValidationError):
+ LanguageValidator.validate("xx")
+
+ def test_unknown_variant_rejected(self):
+ with pytest.raises(ValidationError):
+ LanguageValidator.validate("zz-ZZ")
+
+ def test_empty_rejected(self):
+ with pytest.raises(ValidationError):
+ LanguageValidator.validate("")
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 6f12016..6e822d7 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -35,36 +35,6 @@ from prometheus_client import (
_REPO_ROOT = Path(__file__).resolve().parent.parent
-def _load_metrics_module_with_registry(registry: CollectorRegistry):
- """Load middleware/metrics.py with patched Counter/Histogram to
- use the supplied registry. Returns the loaded module."""
- spec = importlib.util.spec_from_file_location(
- "metrics_under_test",
- _REPO_ROOT / "middleware" / "metrics.py",
- )
- mod = importlib.util.module_from_spec(spec)
- # Inject the fresh registry into the module's namespace before exec
- mod.__dict__["_TEST_REGISTRY"] = registry
-
- # Patch Counter/Histogram to use the fresh registry
- orig_counter = Counter
- orig_histogram = Histogram
-
- def _counter(*args, **kwargs):
- kwargs.setdefault("registry", registry)
- return orig_counter(*args, **kwargs)
-
- def _histogram(*args, **kwargs):
- kwargs.setdefault("registry", registry)
- return orig_histogram(*args, **kwargs)
-
- mod.__dict__["Counter"] = _counter
- mod.__dict__["Histogram"] = _histogram
-
- spec.loader.exec_module(mod)
- return mod
-
-
@pytest.fixture(scope="module")
def metrics():
"""Load the metrics module ONCE per test module.
@@ -80,55 +50,59 @@ def metrics():
return _load_metrics_module_with_fresh_registry()
+def _load_metrics_module_with_registry(registry: CollectorRegistry):
+ """Load middleware/metrics.py with ALL its counters/histograms
+ registered on the supplied fresh registry.
+
+ metrics.py does ``from prometheus_client import Counter, Histogram`` at
+ module top — injecting patched classes into the module dict BEFORE exec
+ does not survive that import. The only reliable interception point is
+ the ``prometheus_client`` module itself: we swap its Counter/Histogram
+ attributes for subclasses that default to ``registry``, exec the
+ module source, and restore the originals.
+ """
+ import types
+ import prometheus_client
+
+ orig_counter = prometheus_client.Counter
+ orig_histogram = prometheus_client.Histogram
+
+ class _RegistryCounter(orig_counter):
+ def __new__(cls, *args, **kwargs):
+ kwargs.setdefault("registry", registry)
+ return super().__new__(cls)
+
+ def __init__(self, *args, **kwargs):
+ kwargs.setdefault("registry", registry)
+ super().__init__(*args, **kwargs)
+
+ class _RegistryHistogram(orig_histogram):
+ def __new__(cls, *args, **kwargs):
+ kwargs.setdefault("registry", registry)
+ return super().__new__(cls)
+
+ def __init__(self, *args, **kwargs):
+ kwargs.setdefault("registry", registry)
+ super().__init__(*args, **kwargs)
+
+ source = (_REPO_ROOT / "middleware" / "metrics.py").read_text(encoding="utf-8")
+ mod = types.ModuleType("metrics_under_test")
+ mod.__file__ = str(_REPO_ROOT / "middleware" / "metrics.py")
+ try:
+ prometheus_client.Counter = _RegistryCounter
+ prometheus_client.Histogram = _RegistryHistogram
+ exec(compile(source, mod.__file__, "exec"), mod.__dict__)
+ finally:
+ prometheus_client.Counter = orig_counter
+ prometheus_client.Histogram = orig_histogram
+ return mod
+
+
def _load_metrics_module_with_fresh_registry():
"""Load metrics module with its counters/histograms attached to a
- fresh CollectorRegistry."""
- spec = importlib.util.spec_from_file_location(
- "metrics_under_test",
- _REPO_ROOT / "middleware" / "metrics.py",
- )
- mod = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(mod)
-
- # The module just created its counters on the default REGISTRY.
- # Unregister them, then re-create them on a fresh registry.
- fresh = CollectorRegistry()
- for name in (
- "http_requests_total",
- "translation_total",
- "translation_duration_seconds",
- "file_size_bytes",
- "quality_l0_checks_total",
- "quality_l1_judge_total",
- "quality_l1_judge_duration_seconds",
- "quality_l1_judge_cost_usd",
- "translation_retry_total",
- "format_elements_lost_total",
- ):
- if not hasattr(mod, name):
- continue
- obj = getattr(mod, name)
- try:
- REGISTRY.unregister(obj)
- except KeyError:
- pass
-
- # Now re-import the module so the new metrics register on `fresh`
- spec = importlib.util.spec_from_file_location(
- "metrics_under_test_isolated",
- _REPO_ROOT / "middleware" / "metrics.py",
- )
- mod2 = importlib.util.module_from_spec(spec)
- # We can't easily re-route Counter/Histogram in exec_module because
- # they call into the global REGISTRY via the function signature.
- # Instead: reload by re-importing via importlib with a wrapper
- # that intercepts the Counter/Histogram constructors. We do this
- # via the more direct route: use the EXISTING counters on the
- # default REGISTRY, but only check RELATIVE increments.
- #
- # Practical approach: just use the module as-is. Tests check
- # `after >= before + 1`, which is robust against other tests.
- return mod
+ fresh CollectorRegistry (never the shared default one — the app may
+ already have registered the same names earlier in the session)."""
+ return _load_metrics_module_with_registry(CollectorRegistry())
def _counter_value(counter, **labels):
diff --git a/tests/test_new_features.py b/tests/test_new_features.py
new file mode 100644
index 0000000..94b2458
--- /dev/null
+++ b/tests/test_new_features.py
@@ -0,0 +1,344 @@
+"""New feature tests: formality/regional prompts, TM, bilingual output,
+QA report, OpenAI JSON batching, font hints."""
+
+import pytest
+from docx import Document
+from openpyxl import Workbook
+
+from services.glossary_service import build_full_prompt
+from services.quality.qa_report import (
+ _number_fidelity,
+ _untranslated_ratio,
+ run_qa_report,
+)
+from services.translation_tm import TMScope, translate_with_tm
+from services.translation_cache import reset_cache_for_tests
+from translators.bilingual import make_bilingual_docx
+from translators.word_translator import WordTranslator, _font_hints_for_target
+
+
+# ===========================================================================
+# Formality + regional variant in prompts
+# ===========================================================================
+class TestFormalityPrompt:
+ def test_formal_adds_tone_directive(self):
+ prompt = build_full_prompt(None, None, "fr", "en", formality="formal")
+ assert "TONE:" in prompt and "formal" in prompt
+
+ def test_informal_adds_tone_directive(self):
+ prompt = build_full_prompt(None, None, "fr", "en", formality="informal")
+ assert "TONE:" in prompt and "informal" in prompt
+
+ def test_no_formality_no_tone(self):
+ assert "TONE:" not in build_full_prompt(None, None, "fr", "en")
+
+ def test_regional_variant_auto(self):
+ prompt = build_full_prompt(None, None, "fr", "pt-BR")
+ assert "REGIONAL VARIANT" in prompt
+ assert "Portuguese" in prompt
+
+ def test_no_region_no_variant(self):
+ assert "REGIONAL VARIANT" not in build_full_prompt(None, None, "fr", "en")
+
+
+# ===========================================================================
+# Translation memory (per-user scoping)
+# ===========================================================================
+class TestTranslationMemory:
+ @pytest.fixture(autouse=True)
+ def clean_cache(self):
+ reset_cache_for_tests()
+ yield
+ reset_cache_for_tests()
+
+ def test_no_scope_passthrough(self):
+ calls = []
+
+ def fake_translate(texts):
+ calls.extend(texts)
+ return [t.upper() for t in texts]
+
+ out = translate_with_tm(
+ ["a", "b"], "en", "fr", "fake", None, fake_translate
+ )
+ assert out == ["A", "B"]
+ assert calls == ["a", "b"]
+
+ def test_reuse_on_second_call_same_user(self):
+ from services.translation_cache import get_cache
+
+ get_cache() # init LRU backend
+
+ calls = []
+
+ def fake_translate(texts):
+ calls.extend(texts)
+ return [f"T:{t}" for t in texts]
+
+ scope = TMScope.from_prompt("user-1", "ctx")
+ first = translate_with_tm(["hello", "world"], "en", "fr", "fake", scope, fake_translate)
+ assert first == ["T:hello", "T:world"]
+ assert len(calls) == 2
+
+ second = translate_with_tm(["hello", "world"], "en", "fr", "fake", scope, fake_translate)
+ assert second == ["T:hello", "T:world"]
+ # Everything came from the TM — no new provider calls
+ assert len(calls) == 2
+
+ def test_users_are_isolated(self):
+ from services.translation_cache import get_cache
+
+ get_cache()
+ out = translate_with_tm(
+ ["x"], "en", "fr", "fake",
+ TMScope.from_prompt("user-A", None), lambda ts: [f"A:{ts[0]}"],
+ )
+ assert out == ["A:x"]
+ out_b = translate_with_tm(
+ ["x"], "en", "fr", "fake",
+ TMScope.from_prompt("user-B", None), lambda ts: [f"B:{ts[0]}"],
+ )
+ assert out_b == ["B:x"]
+
+ def test_identity_translations_not_stored(self):
+ from services.translation_cache import get_cache
+
+ get_cache()
+ scope = TMScope.from_prompt("user-1", None)
+ # identity provider: returns input unchanged (a provider failure)
+ translate_with_tm(["same"], "en", "fr", "fake", scope, lambda ts: ts)
+ out = translate_with_tm(
+ ["same"], "en", "fr", "fake", scope, lambda ts: ["CALLED"]
+ )
+ # Not poisoned by the identity entry: the provider was consulted
+ assert out == ["CALLED"]
+
+ def test_different_prompt_context_misses(self):
+ from services.translation_cache import get_cache
+
+ get_cache()
+ s1 = TMScope.from_prompt("u", "prompt-A")
+ s2 = TMScope.from_prompt("u", "prompt-B")
+ translate_with_tm(["k"], "en", "fr", "fake", s1, lambda ts: ["v1"])
+ out = translate_with_tm(["k"], "en", "fr", "fake", s2, lambda ts: ["v2"])
+ assert out == ["v2"]
+
+
+# ===========================================================================
+# Bilingual output
+# ===========================================================================
+class TestBilingual:
+ def test_source_interleaved_above_translation(self, tmp_path):
+ src = Document()
+ src.add_paragraph("Bonjour le monde")
+ src.add_paragraph("")
+ src.add_paragraph("Deuxième paragraphe")
+ src_in = tmp_path / "src.docx"
+ src.save(str(src_in))
+
+ tr = Document()
+ tr.add_paragraph("Hello world")
+ tr.add_paragraph("")
+ tr.add_paragraph("Second paragraph")
+ tr_out = tmp_path / "tr.docx"
+ tr.save(str(tr_out))
+
+ result = make_bilingual_docx(src_in, tr_out, tmp_path / "bi.docx")
+ assert result is not None
+
+ doc = Document(str(result))
+ texts = [p.text for p in doc.paragraphs]
+ # Source paragraph precedes its translation
+ assert texts[0] == "Bonjour le monde"
+ assert texts[1] == "Hello world"
+ assert texts[3] == "Deuxième paragraphe"
+ assert texts[4] == "Second paragraph"
+
+ def test_structure_mismatch_returns_none(self, tmp_path):
+ a = Document(); a.add_paragraph("x"); a.save(str(tmp_path / "a.docx"))
+ b = Document(); b.add_paragraph("x"); b.add_paragraph("y")
+ b.save(str(tmp_path / "b.docx"))
+ assert make_bilingual_docx(
+ tmp_path / "a.docx", tmp_path / "b.docx", tmp_path / "bi.docx"
+ ) is None
+
+
+# ===========================================================================
+# QA report
+# ===========================================================================
+class TestQAReport:
+ def test_number_fidelity_full(self):
+ r = _number_fidelity("Prix: 12,50 € sur 3 pages", "Price: 12.50 € on 3 pages")
+ assert r["fidelity"] >= 0.9
+
+ def test_number_fidelity_missing(self):
+ r = _number_fidelity("Ref 123 and 456", "Ref 123 only")
+ assert r["fidelity"] < 1.0
+
+ def test_untranslated_ratio_high_when_identity(self):
+ identity_source = (
+ "This document contains substantial english text about pricing, "
+ "delivery schedules and quarterly reporting obligations for the "
+ "regional sales team and their managers."
+ )
+ assert _untranslated_ratio(identity_source, identity_source) > 0.8
+
+ def test_untranslated_ratio_low_when_translated(self):
+ assert _untranslated_ratio(
+ "Ce document contient beaucoup de mots différents sur la facturation, "
+ "les échéances de livraison et les obligations trimestrielles.",
+ "This document holds many different words about invoicing, "
+ "delivery deadlines and quarterly obligations.",
+ ) < 0.35
+
+ def test_run_qa_report_on_docx(self, tmp_path):
+ src = Document()
+ src.add_paragraph("Le total est de 42 euros pour la livraison.")
+ src.add_paragraph("Merci de votre confiance renouvelée.")
+ src_in = tmp_path / "in.docx"
+ src.save(str(src_in))
+
+ out_doc = Document()
+ out_doc.add_paragraph("The total is 42 euros for the delivery.")
+ out_doc.add_paragraph("Thank you for your renewed trust.")
+ out_p = tmp_path / "out.docx"
+ out_doc.save(str(out_p))
+
+ report = run_qa_report(src_in, out_p, "en", ".docx")
+ assert report is not None
+ assert report["score"] >= 80
+ assert report["numbers"]["fidelity"] == 1.0
+
+ def test_run_qa_report_scores_untranslated_low(self, tmp_path):
+ long_fr = (
+ "Ce paragraphe contient suffisamment de mots différents pour "
+ "satisfaire l'heuristique du rapport qualité automatisé, avec "
+ "des notions de facturation, livraison et obligations."
+ )
+ src = Document()
+ src.add_paragraph(long_fr)
+ src_in = tmp_path / "in.docx"
+ src.save(str(src_in))
+ import shutil
+
+ out_p = tmp_path / "out.docx"
+ shutil.copy(str(src_in), str(out_p)) # "translation" = original
+
+ report = run_qa_report(src_in, out_p, "en", ".docx")
+ assert report is not None
+ assert report["score"] < 50
+ assert report["untranslated_ratio"] > 0.5
+
+
+# ===========================================================================
+# OpenAI JSON batching
+# ===========================================================================
+class TestOpenAIBatch:
+ def _provider(self):
+ from services.providers.openai_provider import OpenAITranslationProvider
+
+ return OpenAITranslationProvider(api_key="test-key", model="gpt-test")
+
+ def _requests(self, texts):
+ from services.providers.schemas import TranslationRequest
+
+ return [TranslationRequest(text=t, target_language="fr") for t in texts]
+
+ def test_batch_single_request_success(self, monkeypatch):
+ provider = self._provider()
+ seen = {}
+
+ def fake_api(text, system_prompt):
+ import json
+
+ seen["text"] = text
+ items = json.loads(text)
+ reply = json.dumps(
+ [{"id": it["id"], "translation": f"FR:{it['text']}"} for it in items]
+ )
+ return reply, {}
+
+ monkeypatch.setattr(provider, "_make_api_request", fake_api)
+ out = provider.translate_batch(self._requests(["one", "two", "three"]))
+ assert [r.translated_text for r in out] == [
+ "FR:one", "FR:two", "FR:three",
+ ]
+ # One API call for the whole batch
+ assert isinstance(seen.get("text"), str) and '"id"' in seen["text"]
+
+ def test_batch_falls_back_on_bad_json(self, monkeypatch):
+ provider = self._provider()
+ calls = {"batch": 0, "single": 0}
+
+ def fake_api(text, system_prompt):
+ if text.startswith("["):
+ calls["batch"] += 1
+ return "not valid json at all", {}
+ calls["single"] += 1
+ return f"FR:{text}", {}
+
+ monkeypatch.setattr(provider, "_make_api_request", fake_api)
+ out = provider.translate_batch(self._requests(["alpha", "beta"]))
+ assert [r.translated_text for r in out] == ["FR:alpha", "FR:beta"]
+ assert calls["batch"] == 1
+ assert calls["single"] == 2
+
+ def test_batch_falls_back_on_wrong_length(self, monkeypatch):
+ import json
+
+ provider = self._provider()
+
+ def fake_api(text, system_prompt):
+ return json.dumps([{"id": 0, "translation": "only one"}]), {}
+
+ monkeypatch.setattr(provider, "_make_api_request", fake_api)
+
+ def fail_single(req):
+ from services.providers.schemas import TranslationResponse
+
+ return TranslationResponse(
+ translated_text=f"S:{req.text}", provider_name="openai"
+ )
+
+ monkeypatch.setattr(provider, "translate_text", fail_single)
+ out = provider.translate_batch(self._requests(["a", "b"]))
+ assert [r.translated_text for r in out] == ["S:a", "S:b"]
+
+
+# ===========================================================================
+# CJK font hints (Word)
+# ===========================================================================
+class TestFontHints:
+ def test_hint_mapping(self):
+ assert _font_hints_for_target("zh-CN")[0] == "SimSun"
+ assert _font_hints_for_target("ja")[0] == "Yu Mincho"
+ assert _font_hints_for_target("ar")[1] == "Arial"
+ assert _font_hints_for_target("en") == (None, None)
+
+ def test_applied_on_translate(self, tmp_path):
+ class _CJK:
+ def get_name(self):
+ return "mock"
+
+ def is_available(self):
+ return True
+
+ def translate_batch(self, texts, target_language, source_language="auto"):
+ return [f"译:{t}" for t in texts]
+
+ from docx.oxml.ns import qn as _qn
+
+ doc = Document()
+ doc.add_paragraph("Hello")
+ src = tmp_path / "in.docx"
+ doc.save(str(src))
+
+ t = WordTranslator(provider=_CJK())
+ out = tmp_path / "out.docx"
+ t.translate_file(src, out, "zh-CN", "en")
+
+ result = Document(str(out))
+ para = result.paragraphs[0]
+ rFonts = para.runs[0]._r.find(_qn("w:rPr")).find(_qn("w:rFonts"))
+ assert rFonts is not None
+ assert rFonts.get(_qn("w:eastAsia")) == "SimSun"
diff --git a/tests/test_plan_gating.py b/tests/test_plan_gating.py
new file mode 100644
index 0000000..23b42ca
--- /dev/null
+++ b/tests/test_plan_gating.py
@@ -0,0 +1,86 @@
+"""Plan-based engine gating and the expanded /languages endpoint."""
+
+import pytest
+
+from models.subscription import PlanType
+from routes.translate_routes import (
+ _allowed_providers_for_plan,
+ _image_translation_allowed_for_plan,
+ _plan_from_user,
+)
+
+
+class _FakeUser:
+ def __init__(self, plan):
+ self.plan = plan
+
+
+class TestAllowedProviders:
+ def test_free_gets_google_only(self):
+ assert _allowed_providers_for_plan(PlanType.FREE) == {"google"}
+
+ def test_starter_adds_deepl(self):
+ assert _allowed_providers_for_plan(PlanType.STARTER) == {"google", "deepl"}
+
+ def test_pro_adds_cloud_and_openrouter(self):
+ allowed = _allowed_providers_for_plan(PlanType.PRO)
+ assert {"google_cloud", "openrouter"} <= allowed
+ assert "openai" not in allowed
+
+ def test_business_adds_premium_and_xai(self):
+ allowed = _allowed_providers_for_plan(PlanType.BUSINESS)
+ assert {"openrouter_premium", "openai", "zai"} <= allowed
+
+ def test_enterprise_has_all(self):
+ enterprise = _allowed_providers_for_plan(PlanType.ENTERPRISE)
+ assert enterprise >= _allowed_providers_for_plan(PlanType.BUSINESS)
+
+
+class TestImageTranslationGate:
+ @pytest.mark.parametrize("plan", [PlanType.FREE, PlanType.STARTER])
+ def test_refused_below_pro(self, plan):
+ assert _image_translation_allowed_for_plan(plan) is False
+
+ @pytest.mark.parametrize("plan", [PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE])
+ def test_allowed_from_pro(self, plan):
+ assert _image_translation_allowed_for_plan(plan) is True
+
+
+class TestPlanFromUser:
+ def test_anonymous_is_free(self):
+ assert _plan_from_user(None) is PlanType.FREE
+
+ def test_enum_plan_passthrough(self):
+ assert _plan_from_user(_FakeUser(PlanType.PRO)) is PlanType.PRO
+
+ def test_string_plan_accepted(self):
+ assert _plan_from_user(_FakeUser("pro")) is PlanType.PRO
+
+ def test_garbage_falls_back_to_free(self):
+ assert _plan_from_user(_FakeUser("nonsense")) is PlanType.FREE
+
+
+class TestLanguagesEndpoint:
+ @pytest.mark.asyncio
+ async def test_exposes_at_least_60_languages(self):
+ from routes.legacy_routes import get_supported_languages
+
+ response = await get_supported_languages()
+ langs = response["supported_languages"]
+ assert response["count"] >= 60
+ assert len(langs) >= 60
+
+ @pytest.mark.asyncio
+ async def test_no_auto_and_canonical_chinese(self):
+ from routes.legacy_routes import get_supported_languages
+
+ langs = (await get_supported_languages())["supported_languages"]
+ assert "auto" not in langs
+ assert "zh-CN" in langs and "zh-TW" in langs
+
+ @pytest.mark.asyncio
+ async def test_every_language_has_a_name(self):
+ from routes.legacy_routes import get_supported_languages
+
+ langs = (await get_supported_languages())["supported_languages"]
+ assert all(name and name != code.upper() for code, name in langs.items())
diff --git a/tests/test_providers/test_minimax_provider.py b/tests/test_providers/test_minimax_provider.py
new file mode 100644
index 0000000..3ab563a
--- /dev/null
+++ b/tests/test_providers/test_minimax_provider.py
@@ -0,0 +1,161 @@
+"""
+Tests for the MinimaxTranslationProvider.
+
+Validates Bug 1 fix:
+- default base_url is the public ``api.minimax.io`` host (NOT ``api.minimax.chat``)
+- default model is ``MiniMax-M3``
+- ``is_available()`` / ``health_check()`` tolerate a missing ``/models`` path
+ (Minimax does not document it) and only mark the provider down on 401/network error
+- translation success, 429 retry, 401 handling
+"""
+
+import pytest
+from unittest.mock import patch, MagicMock
+from requests.exceptions import Timeout
+
+from services.providers.minimax_provider import (
+ MinimaxTranslationProvider,
+ MinimaxProviderError,
+ MINIMAX_RATE_LIMITED,
+ MINIMAX_INVALID_KEY,
+ MINIMAX_TIMEOUT,
+ MINIMAX_SERVICE_ERROR,
+)
+from services.providers.schemas import TranslationRequest
+
+
+class TestMinimaxProviderConfig:
+ """Defaults must point at the real public endpoint."""
+
+ def test_default_base_url_is_public_host(self):
+ provider = MinimaxTranslationProvider(api_key="k", max_retries=0)
+ assert provider._base_url == "https://api.minimax.io/v1"
+ # Regression guard: the old broken host must never come back.
+ assert "minimax.chat" not in provider._base_url
+
+ def test_default_model_is_m3(self):
+ provider = MinimaxTranslationProvider(api_key="k", max_retries=0)
+ assert provider._model == "MiniMax-M3"
+
+ def test_custom_base_url_respected(self):
+ provider = MinimaxTranslationProvider(
+ api_key="k", base_url="https://proxy.example.com/v1", max_retries=0
+ )
+ assert provider._base_url == "https://proxy.example.com/v1"
+
+ def test_get_name(self):
+ provider = MinimaxTranslationProvider(api_key="k", max_retries=0)
+ assert provider.get_name() == "minimax"
+
+
+class TestMinimaxAvailabilityProbe:
+ """is_available/health_check must not fail just because /models 404s."""
+
+ @pytest.fixture
+ def provider(self):
+ return MinimaxTranslationProvider(api_key="k", max_retries=0)
+
+ def _mock_get(self, status_code: int):
+ mock_response = MagicMock()
+ mock_response.status_code = status_code
+ return mock_response
+
+ @patch("requests.get")
+ def test_available_when_models_returns_200(self, mock_get, provider):
+ mock_get.return_value = self._mock_get(200)
+ assert provider.is_available() is True
+
+ @patch("requests.get")
+ def test_available_when_models_404(self, mock_get, provider):
+ # Minimax does not document /models; a 404 must NOT mark it unavailable.
+ mock_get.return_value = self._mock_get(404)
+ assert provider.is_available() is True
+
+ @patch("requests.get")
+ def test_unavailable_on_401(self, mock_get, provider):
+ mock_get.return_value = self._mock_get(401)
+ assert provider.is_available() is False
+
+ @patch("requests.get")
+ def test_unavailable_on_network_error(self, _mock_get, provider):
+ def _raise(*a, **kw):
+ raise Timeout("boom")
+
+ with patch("requests.get", side_effect=_raise):
+ assert provider.is_available() is False
+
+ @patch("requests.get")
+ def test_health_check_tolerates_404(self, mock_get, provider):
+ mock_get.return_value = self._mock_get(404)
+ status = provider.health_check()
+ assert status.available is True
+ assert status.name == "minimax"
+
+ @patch("requests.get")
+ def test_health_check_marks_down_on_401(self, mock_get, provider):
+ mock_get.return_value = self._mock_get(401)
+ status = provider.health_check()
+ assert status.available is False
+
+
+class TestMinimaxTranslateText:
+ @pytest.fixture
+ def provider(self):
+ return MinimaxTranslationProvider(api_key="k", model="MiniMax-M3", max_retries=0)
+
+ def _mock_post(self, payload, status_code=200):
+ mock_response = MagicMock()
+ mock_response.status_code = status_code
+ mock_response.json.return_value = payload
+ mock_response.text = ""
+ return mock_response
+
+ @patch("requests.post")
+ def test_success(self, mock_post, provider):
+ mock_post.return_value = self._mock_post(
+ {"choices": [{"message": {"content": "Bonjour"}}], "usage": {}}
+ )
+ resp = provider.translate_text(TranslationRequest(text="Hello", target_language="fr"))
+ assert resp.translated_text == "Bonjour"
+ assert resp.provider_name == "minimax"
+ # Verify we hit the public host on the OpenAI-compatible path.
+ called_url = mock_post.call_args[0][0]
+ assert called_url == "https://api.minimax.io/v1/chat/completions"
+
+ def test_empty_text_short_circuits(self, provider):
+ resp = provider.translate_text(TranslationRequest(text="", target_language="fr"))
+ assert resp.translated_text == ""
+
+ @patch("requests.post")
+ def test_invalid_key_returns_error(self, mock_post, provider):
+ mock_post.return_value = self._mock_post({"error": "bad key"}, status_code=401)
+ resp = provider.translate_text(TranslationRequest(text="Hello", target_language="fr"))
+ assert resp.error_code == MINIMAX_INVALID_KEY
+ # Original text returned on failure.
+ assert resp.translated_text == "Hello"
+
+ @patch("time.sleep")
+ @patch("requests.post")
+ def test_rate_limit_then_success(self, mock_post, mock_sleep):
+ provider = MinimaxTranslationProvider(api_key="k", max_retries=2, retry_delay=0.01)
+ mock_post.side_effect = [
+ self._mock_post({"error": "slow down"}, status_code=429),
+ self._mock_post({"choices": [{"message": {"content": "Hola"}}], "usage": {}}),
+ ]
+ resp = provider.translate_text(TranslationRequest(text="Hello", target_language="es"))
+ assert resp.translated_text == "Hola"
+ assert mock_sleep.called # backoff happened
+
+ @patch("requests.post")
+ def test_service_error_when_empty_choices(self, mock_post, provider):
+ mock_post.return_value = self._mock_post({"choices": []})
+ resp = provider.translate_text(TranslationRequest(text="Hello", target_language="fr"))
+ assert resp.error_code == MINIMAX_SERVICE_ERROR
+
+
+class TestMinimaxProviderError:
+ def test_error_carries_code_and_message(self):
+ err = MinimaxProviderError(MINIMAX_TIMEOUT, "timed out", details={"wait": 1})
+ assert err.code == MINIMAX_TIMEOUT
+ assert err.message == "timed out"
+ assert err.details == {"wait": 1}
diff --git a/tests/test_providers/test_openai_provider.py b/tests/test_providers/test_openai_provider.py
index d426d7c..feb6cfe 100644
--- a/tests/test_providers/test_openai_provider.py
+++ b/tests/test_providers/test_openai_provider.py
@@ -97,11 +97,24 @@ class TestHelperFunctions:
assert "translator" in prompt.lower()
def test_build_system_prompt_custom(self):
- """Test custom system prompt."""
+ """A custom prompt AUGMENTS the base translation instructions.
+
+ The base prompt is always present — a glossary-only custom prompt
+ used to produce a system prompt with no translation instruction
+ at all (fixed 2026-08-29).
+ """
custom = "Translate this text formally for business context."
prompt = _build_system_prompt("English", "French", custom)
- assert prompt == custom
+ # Base translation instructions survive
+ assert "English" in prompt
+ assert "French" in prompt
+ assert "translator" in prompt.lower()
+ # Custom content is appended, not replacing
+ assert custom in prompt
+ assert "ADDITIONAL CONTEXT AND INSTRUCTIONS" in prompt
+ # The base part comes first
+ assert prompt.index("French") < prompt.index(custom)
class TestOpenAITranslationProvider:
diff --git a/tests/test_scanned_pdf_ocr.py b/tests/test_scanned_pdf_ocr.py
new file mode 100644
index 0000000..75e2dc5
--- /dev/null
+++ b/tests/test_scanned_pdf_ocr.py
@@ -0,0 +1,192 @@
+"""Scanned PDF detection and the Mistral OCR translation path."""
+
+import fitz
+import pytest
+
+from config import config
+from services.mistral_ocr import MistralOCRClient
+from services.providers.base import TranslationProvider
+from services.providers.schemas import TranslationRequest, TranslationResponse
+from translators.pdf_translator import PDFTranslator
+
+
+FAKE_OCR_PAGES = [
+ "# Facture\n\nMontant total : 1 250 euros\n\n",
+ "Le paiement est du sous 30 jours.",
+]
+
+# Minimal French → English mapping for the fake provider; anything else is
+# prefixed so tests can assert replacement happened.
+TRANSLATIONS = {
+ "Facture": "Invoice",
+ "Montant total : 1 250 euros": "Total amount: 1,250 euros",
+ "Le paiement est du sous 30 jours.": "Payment is due within 30 days.",
+}
+
+
+class FakeProvider(TranslationProvider):
+ """New-style provider with canned translations (no network)."""
+
+ def get_name(self) -> str:
+ return "fake"
+
+ def is_available(self) -> bool:
+ return True
+
+ def translate_text(self, request: TranslationRequest) -> TranslationResponse:
+ # Pages arrive as multi-line text: translate line by line so the
+ # canned mapping applies regardless of how lines are grouped.
+ lines = []
+ for line in request.text.split("\n"):
+ stripped = line.strip()
+ if not stripped:
+ lines.append(line)
+ continue
+ lines.append(TRANSLATIONS.get(stripped, f"[EN] {stripped}"))
+ return TranslationResponse(
+ translated_text="\n".join(lines),
+ provider_name=self.get_name(),
+ from_cache=False,
+ )
+
+
+def _make_scanned_pdf(path):
+ """Image-only PDF: one page almost fully covered by a picture, no text."""
+ pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 10, 10))
+ pix.clear_with(90)
+ png_bytes = pix.tobytes("png")
+ doc = fitz.open()
+ page = doc.new_page(width=612, height=792)
+ page.insert_image(fitz.Rect(20, 20, 592, 772), stream=png_bytes)
+ doc.save(str(path))
+ doc.close()
+ return path
+
+
+def _make_text_pdf(path):
+ doc = fitz.open()
+ page = doc.new_page()
+ page.insert_text(
+ (72, 100),
+ "Real selectable text with plenty of characters to exceed the scanned threshold. " * 2,
+ fontsize=11,
+ )
+ doc.save(str(path))
+ doc.close()
+ return path
+
+
+@pytest.fixture
+def no_api_key(monkeypatch):
+ monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
+ monkeypatch.setattr(config, "MISTRAL_API_KEY", "")
+ monkeypatch.setattr(config, "MISTRAL_OCR_ENABLED", True)
+
+
+@pytest.fixture
+def api_key(monkeypatch):
+ # Set BOTH the env var and the config attribute: some other test in the
+ # full suite reloads the config module, and pdf_translator imports
+ # `config` at call time — after a reload only the env var is still
+ # visible (the class attributes are re-read from the environment).
+ monkeypatch.setenv("MISTRAL_API_KEY", "test-key")
+ monkeypatch.setattr(config, "MISTRAL_API_KEY", "test-key")
+ monkeypatch.setattr(config, "MISTRAL_OCR_ENABLED", True)
+
+
+class TestScannedDetection:
+ def test_image_only_pdf_is_scanned(self, tmp_path):
+ pdf = _make_scanned_pdf(tmp_path / "scan.pdf")
+ translator = PDFTranslator(provider=None)
+ assert translator._is_scanned_pdf(pdf) is True
+
+ def test_text_pdf_is_not_scanned(self, tmp_path):
+ pdf = _make_text_pdf(tmp_path / "text.pdf")
+ translator = PDFTranslator(provider=None)
+ assert translator._is_scanned_pdf(pdf) is False
+
+ def test_title_only_pdf_is_not_scanned(self, tmp_path):
+ """A sparse but textual page (no raster image) stays on the layout path."""
+ doc = fitz.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Facture 2026", fontsize=18)
+ doc.save(str(tmp_path / "title.pdf"))
+ doc.close()
+ translator = PDFTranslator(provider=None)
+ assert translator._is_scanned_pdf(tmp_path / "title.pdf") is False
+
+
+class TestMarkdownCleanup:
+ def test_images_links_headings_removed(self):
+ md = "# Titre\n\n\n\n[lien](http://x) texte"
+ out = PDFTranslator._markdown_to_text(md)
+ assert "Titre" in out
+ assert "![img]" not in out
+ assert "(http://x)" not in out
+ assert "#" not in out
+ assert "lien texte" in out
+
+ def test_table_pipes_removed(self):
+ out = PDFTranslator._markdown_to_text("| A | B |\n|---|---|\n| un | deux |")
+ assert "|" not in out
+ assert "un" in out and "deux" in out
+
+
+class TestScannedOCRPath:
+ def test_translate_scanned_pdf_end_to_end(self, tmp_path, api_key, monkeypatch):
+ pdf = _make_scanned_pdf(tmp_path / "scan.pdf")
+ monkeypatch.setattr(
+ MistralOCRClient,
+ "extract_pdf_text",
+ lambda self, path, progress_callback=None: list(FAKE_OCR_PAGES),
+ )
+
+ translator = PDFTranslator(provider=FakeProvider())
+ # Configure OCR the way the production route does (set_ocr_config).
+ # Going through the config module is fragile in the full suite:
+ # another test replaces sys.modules["config"], so a config-module
+ # patch may never reach the translator.
+ translator.set_ocr_config(api_key="test-key", enabled=True)
+ out = tmp_path / "out.pdf"
+ result = translator.translate_file(pdf, out, "en", "fr")
+
+ assert result.exists() and result.stat().st_size > 0
+ doc = fitz.open(str(result))
+ text = "\n".join(p.get_text("text") for p in doc)
+ doc.close()
+
+ assert "Invoice" in text
+ assert "Payment is due within 30 days" in text
+ assert "Facture" not in text
+ assert "logo" not in text # markdown images stripped
+ assert "|" not in text # markdown tables stripped
+
+ def test_missing_api_key_raises_clear_error(self, tmp_path, no_api_key):
+ pdf = _make_scanned_pdf(tmp_path / "scan.pdf")
+ translator = PDFTranslator(provider=FakeProvider())
+ with pytest.raises(RuntimeError) as exc:
+ translator.translate_file(pdf, tmp_path / "out.pdf", "en", "fr")
+ assert "MISTRAL_API_KEY" in str(exc.value)
+
+ def test_disabled_ocr_raises_clear_error(self, tmp_path, monkeypatch):
+ monkeypatch.setattr(config, "MISTRAL_API_KEY", "test-key")
+ monkeypatch.setattr(config, "MISTRAL_OCR_ENABLED", False)
+ pdf = _make_scanned_pdf(tmp_path / "scan.pdf")
+ translator = PDFTranslator(provider=FakeProvider())
+ with pytest.raises(RuntimeError):
+ translator.translate_file(pdf, tmp_path / "out.pdf", "en", "fr")
+
+ def test_text_pdf_bypasses_ocr(self, tmp_path, api_key, monkeypatch):
+ pdf = _make_text_pdf(tmp_path / "text.pdf")
+ called = {"ocr": False}
+
+ def _fail(self, path, progress_callback=None):
+ called["ocr"] = True
+ return []
+
+ monkeypatch.setattr(MistralOCRClient, "extract_pdf_text", _fail)
+
+ translator = PDFTranslator(provider=FakeProvider())
+ out = tmp_path / "out.pdf"
+ translator.translate_file(pdf, out, "en", "fr")
+ assert called["ocr"] is False
diff --git a/tests/test_security_fixes_c1_c4.py b/tests/test_security_fixes_c1_c4.py
new file mode 100644
index 0000000..6f6c85a
--- /dev/null
+++ b/tests/test_security_fixes_c1_c4.py
@@ -0,0 +1,228 @@
+"""Tests for security fixes C1–C4 (audit 2026-08-26)."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from routes import translate_routes as tr
+from middleware.cleanup import FileCleanupManager
+
+
+class TestSanitizeUrlFilename:
+ """AC1 — path traversal in URL-downloaded filenames is neutralized."""
+
+ @pytest.mark.parametrize(
+ "raw",
+ [
+ "../../evil.xlsx",
+ "..\\..\\evil.docx",
+ "normal_file.pptx",
+ "..\t..evil.pdf",
+ "...",
+ "",
+ ],
+ )
+ def test_traversal_stripped(self, raw):
+ result = tr._sanitize_url_filename(raw)
+ # Core guarantee: no traversal or path separators survive
+ assert ".." not in result
+ assert "/" not in result
+ assert "\\" not in result
+ assert result != ""
+
+ def test_long_filename_truncated(self):
+ raw = "a" * 300 + ".xlsx"
+ result = tr._sanitize_url_filename(raw)
+ assert len(result) <= 255
+ assert result.endswith(".xlsx")
+
+
+class TestRedirectSsrf:
+ """AC2 — redirect to internal address is blocked."""
+
+ @pytest.mark.asyncio
+ async def test_redirect_to_metadata_blocked(self):
+ def fake_response(status, location=None):
+ resp = MagicMock()
+ resp.status_code = status
+ resp.headers = {"location": location} if location else {}
+ return resp
+
+ class FakeClient:
+ def build_request(self, method, url):
+ return (method, url)
+
+ async def send(self, req, stream=False):
+ url = req[1]
+ if "public.example" in url:
+ return fake_response(302, "http://169.254.169.254/latest/meta-data")
+ return fake_response(200)
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *a):
+ return False
+
+ with patch.object(tr.httpx, "AsyncClient", return_value=FakeClient()):
+ with pytest.raises(tr.TranslateEndpointError) as exc:
+ await tr.download_from_url("http://public.example/file.xlsx")
+ assert exc.value.details.get("reason") == "ssrf_blocked"
+
+
+class TestCleanupOrphanGrace:
+ """AC3 — young orphans are not deleted."""
+
+ def _manager(self, tmp_path):
+ m = FileCleanupManager(
+ upload_dir=tmp_path / "uploads",
+ output_dir=tmp_path / "outputs",
+ temp_dir=tmp_path / "temp",
+ )
+ for d in (m.upload_dir, m.output_dir, m.temp_dir):
+ d.mkdir(parents=True, exist_ok=True)
+ return m
+
+ @pytest.mark.asyncio
+ async def test_young_orphan_kept(self, tmp_path):
+ """Orphan younger than the grace period is NOT deleted."""
+ m = self._manager(tmp_path)
+ f = m.upload_dir / "recent_orphan.xlsx"
+ f.write_bytes(b"x")
+
+ fake_redis = MagicMock()
+ fake_redis.keys = AsyncMock(return_value=[])
+ fake_redis.get = AsyncMock(return_value=None)
+ with patch(
+ "middleware.cleanup._get_async_redis", return_value=fake_redis
+ ):
+ stats = await m.cleanup()
+ assert f.exists()
+
+ @pytest.mark.asyncio
+ async def test_input_path_key_recognized(self, tmp_path):
+ """Files tracked under 'input_path' are not orphans (key mismatch fix)."""
+ import json as _json
+
+ m = self._manager(tmp_path)
+ f = m.upload_dir / "tracked.xlsx"
+ f.write_bytes(b"x")
+
+ fake_redis = MagicMock()
+ fake_redis.keys = AsyncMock(return_value=["translation:file:tr_1"])
+ fake_redis.get = AsyncMock(
+ return_value=_json.dumps({"input_path": str(f), "user_id": "u1"})
+ )
+ with patch(
+ "middleware.cleanup._get_async_redis", return_value=fake_redis
+ ):
+ stats = await m.cleanup()
+ assert f.exists()
+ assert stats["orphaned_deleted"] == 0
+
+ @pytest.mark.asyncio
+ async def test_old_orphan_deleted(self, tmp_path):
+ import json as _json
+ import os
+ import time
+
+ m = self._manager(tmp_path)
+ f = m.upload_dir / "old_orphan.xlsx"
+ f.write_bytes(b"x")
+ old = time.time() - (m.orphan_grace_seconds + 600)
+ os.utime(f, (old, old))
+
+ fake_redis = MagicMock()
+ fake_redis.keys = AsyncMock(return_value=[])
+ fake_redis.get = AsyncMock(return_value=None)
+ with patch(
+ "middleware.cleanup._get_async_redis", return_value=fake_redis
+ ):
+ stats = await m.cleanup()
+ assert not f.exists()
+ assert stats["orphaned_deleted"] == 1
+
+
+class TestJobAccessControl:
+ """H2 — ownership / token checks on status and download."""
+
+ def _job(self, user_id=None, token="tok123"):
+ return {"id": "tr_abc", "user_id": user_id, "access_token": token}
+
+ def _user(self, uid):
+ u = MagicMock()
+ u.id = uid
+ return u
+
+ def test_owner_allowed(self):
+ job = self._job(user_id=7)
+ assert tr._check_job_access(job, self._user(7), None) is None
+
+ def test_other_user_denied(self):
+ job = self._job(user_id=7)
+ resp = tr._check_job_access(job, self._user(8), None)
+ assert resp is not None and resp.status_code == 403
+
+ def test_anonymous_caller_on_owned_job_denied(self):
+ job = self._job(user_id=7)
+ resp = tr._check_job_access(job, None, "tok123")
+ assert resp is not None and resp.status_code == 401
+
+ def test_anonymous_job_requires_token(self):
+ job = self._job(user_id=None)
+ assert tr._check_job_access(job, None, "wrong") is not None
+ assert tr._check_job_access(job, None, None) is not None
+ assert tr._check_job_access(job, None, "tok123") is None
+
+ def test_old_anonymous_job_without_token_denied(self):
+ job = {"id": "tr_old", "user_id": None} # job created before the fix
+ assert tr._check_job_access(job, None, "anything") is not None
+
+
+class TestZipBomb:
+ """H1 — dangerous archives are rejected."""
+
+ def _make_zip(self, tmp_path, entries):
+ import zipfile
+
+ p = tmp_path / "bomb.xlsx"
+ with zipfile.ZipFile(p, "w", zipfile.ZIP_DEFLATED) as zf:
+ for name, data in entries:
+ zf.writestr(name, data)
+ return p
+
+ def test_normal_file_accepted(self, tmp_path):
+ from utils.file_handler import validate_zip_safety
+
+ p = self._make_zip(tmp_path, [("sheet1.xml", b"ok" * 100)])
+ validate_zip_safety(p) # no exception
+
+ def test_not_a_zip_rejected(self, tmp_path):
+ from utils.file_handler import validate_zip_safety
+
+ p = tmp_path / "fake.xlsx"
+ p.write_bytes(b"this is not a zip file")
+ with pytest.raises(ValueError):
+ validate_zip_safety(p)
+
+ def test_high_ratio_rejected(self, tmp_path):
+ from utils.file_handler import validate_zip_safety
+
+ # 50 MB of zeros compresses far beyond the 100:1 ratio cap
+ p = self._make_zip(tmp_path, [("huge.xml", b"\0" * (50 * 1024 * 1024))])
+ with pytest.raises(ValueError):
+ validate_zip_safety(p)
+
+ def test_declared_total_too_big_rejected(self, tmp_path):
+ import zipfile
+ from unittest.mock import patch as _patch
+ from utils.file_handler import validate_zip_safety
+
+ p = self._make_zip(tmp_path, [("a.xml", b"")])
+ fake_info = MagicMock()
+ fake_info.is_dir = lambda: False
+ fake_info.file_size = 5 * 1024 * 1024 * 1024 # 5 GB declared
+ fake_info.compress_size = 50 * 1024 * 1024
+ with _patch.object(zipfile.ZipFile, "infolist", return_value=[fake_info]):
+ with pytest.raises(ValueError):
+ validate_zip_safety(p)
diff --git a/tests/test_story_2_13_validation.py b/tests/test_story_2_13_validation.py
index cf74bc1..7787248 100644
--- a/tests/test_story_2_13_validation.py
+++ b/tests/test_story_2_13_validation.py
@@ -30,10 +30,18 @@ def test_validate_invalid_magic_bytes():
assert response.json()["error"] == "CORRUPTED_FILE"
assert "corrompu" in response.json()["message"]
+def _minimal_zip() -> bytes:
+ """A real minimal ZIP archive (Office files are ZIPs)."""
+ import io, zipfile
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ zf.writestr("[Content_Types].xml", "")
+ return buf.getvalue()
+
+
def test_validate_valid_file_header():
- # Test with a minimal valid-looking zip (Office files are ZIPs)
- # FileValidator checks for b"PK\x03\x04"
- files = {"file": ("test.docx", b"PK\x03\x04" + b"\x00" * 20, "application/vnd.openxmlformats-officedocument.wordprocessingml.document")}
+ # Minimal real ZIP: passes magic-byte AND zip-safety checks
+ files = {"file": ("test.docx", _minimal_zip(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document")}
response = client.post(
"/api/v1/translate",
files=files,
diff --git a/tests/test_translate_endpoint.py b/tests/test_translate_endpoint.py
index 95379cb..fc8875c 100644
--- a/tests/test_translate_endpoint.py
+++ b/tests/test_translate_endpoint.py
@@ -593,7 +593,11 @@ class TestOptionalParameters:
assert response.status_code == 202
def test_accepts_mode_llm(self, authenticated_client):
- """Accepts mode='llm'"""
+ """mode='llm' maps to the openrouter engine — Pro and above only.
+
+ The default test user is on the Free plan, so the plan-based engine
+ gate must refuse it (403) instead of silently running a paid engine.
+ """
excel_content = create_valid_excel()
response = authenticated_client.post(
TRANSLATE_URL,
@@ -606,7 +610,8 @@ class TestOptionalParameters:
},
data={"target_lang": "fr", "mode": "llm"},
)
- assert response.status_code == 202
+ assert response.status_code == 403
+ assert "PRO_FEATURE_REQUIRED" in str(response.json())
def test_accepts_webhook_url(self, authenticated_client):
"""Accepts webhook_url parameter"""
@@ -875,7 +880,11 @@ class TestTranslateImagesParameter:
"""Test translate_images parameter in POST /api/v1/translate"""
def test_accepts_translate_images_parameter(self, authenticated_client):
- """Endpoint accepts translate_images form parameter"""
+ """translate_images is a Pro+ feature — Free plan gets 403.
+
+ The parameter is accepted by the schema; the plan gate refuses it
+ for the default (Free) test user.
+ """
excel_content = create_valid_excel()
response = authenticated_client.post(
TRANSLATE_URL,
@@ -888,4 +897,5 @@ class TestTranslateImagesParameter:
},
data={"target_lang": "fr", "translate_images": "true"},
)
- assert response.status_code == 202
+ assert response.status_code == 403
+ assert "PRO_FEATURE_REQUIRED" in str(response.json())
diff --git a/tests/test_translate_routes_helpers.py b/tests/test_translate_routes_helpers.py
new file mode 100644
index 0000000..b48f855
--- /dev/null
+++ b/tests/test_translate_routes_helpers.py
@@ -0,0 +1,168 @@
+"""
+Unit tests for the helper functions in routes/translate_routes.py.
+
+These cover the bug fixes that are pure logic (no full HTTP machinery needed):
+- Bug 2: _compute_cost_factor / _provider_model (reads ``_model`` AND ``model``)
+- Bug 3: _release_quota_if_needed (releases reserved quota on soft-failure)
+- Bug 4: _compute_duration_seconds (UTC-aware, never crashes)
+- Bug 5: _cleanup_old_jobs snapshots the jobs dict (no "changed size during iteration")
+"""
+
+import asyncio
+import time
+from datetime import datetime, timezone, timedelta
+from unittest.mock import patch, MagicMock
+
+import pytest
+
+from routes import translate_routes as tr
+
+
+# ---------------------------------------------------------------------------
+# Bug 2 — _provider_model / _compute_cost_factor
+# ---------------------------------------------------------------------------
+class _NewStyleProvider:
+ """Mimics services/providers/* classes that store ``self._model``."""
+
+ def __init__(self, model):
+ self._model = model
+
+
+class _LegacyProvider:
+ """Mimics legacy services/translation_service classes with ``self.model``."""
+
+ def __init__(self, model):
+ self.model = model
+
+
+class TestProviderModel:
+ def test_reads_new_style_private_attr(self):
+ assert tr._provider_model(_NewStyleProvider("gpt-4o")) == "gpt-4o"
+
+ def test_reads_legacy_public_attr(self):
+ assert tr._provider_model(_LegacyProvider("claude-sonnet-4")) == "claude-sonnet-4"
+
+ def test_prefers_private_when_both_present(self):
+ class Both:
+ _model = "private-model"
+ model = "public-model"
+
+ assert tr._provider_model(Both()) == "private-model"
+
+ def test_none_provider(self):
+ assert tr._provider_model(None) == ""
+
+ def test_empty_model(self):
+ assert tr._provider_model(_NewStyleProvider("")) == ""
+
+
+class TestComputeCostFactor:
+ def test_claude_is_premium(self):
+ assert tr._compute_cost_factor(_NewStyleProvider("anthropic/claude-sonnet-4.6")) == 5
+
+ def test_gpt4_is_premium(self):
+ assert tr._compute_cost_factor(_NewStyleProvider("gpt-4o")) == 5
+
+ def test_gpt4o_mini_is_standard(self):
+ # Cheap GPT-4 variants must not be billed at the premium factor.
+ assert tr._compute_cost_factor(_NewStyleProvider("gpt-4o-mini")) == 1
+ assert tr._compute_cost_factor(_NewStyleProvider("gpt-4o-nano")) == 1
+
+ def test_legacy_gpt4o_mini_is_standard(self):
+ assert tr._compute_cost_factor(_LegacyProvider("gpt-4o-mini")) == 1
+
+ def test_haiku_is_standard(self):
+ assert tr._compute_cost_factor(_NewStyleProvider("anthropic/claude-3-haiku")) == 1
+
+ def test_openrouter_premium_alias_is_premium_without_model(self):
+ # Regression for the original bug: model read failed (""), so the
+ # premium tier was never detected. The alias must still bump it to 5.
+ assert tr._compute_cost_factor(None, "openrouter_premium") == 5
+
+ def test_standard_model(self):
+ assert tr._compute_cost_factor(_NewStyleProvider("deepseek-chat")) == 1
+
+ def test_legacy_provider_model_is_read(self):
+ # Legacy classes expose .model — must also be billed correctly.
+ assert tr._compute_cost_factor(_LegacyProvider("gpt-4o")) == 5
+
+
+# ---------------------------------------------------------------------------
+# Bug 3 — _release_quota_if_needed
+# ---------------------------------------------------------------------------
+class TestReleaseQuotaIfNeeded:
+ @pytest.mark.asyncio
+ async def test_releases_when_user_id_and_not_recorded(self):
+ # release_translation_quota is invoked via asyncio.to_thread (a worker
+ # thread); patching it and asserting the call validates the release path.
+ with patch.object(tr, "release_translation_quota") as mock_release:
+ await tr._release_quota_if_needed("user-123", usage_recorded=False, job_id="j1")
+ mock_release.assert_called_once_with("user-123")
+
+ @pytest.mark.asyncio
+ async def test_skips_when_usage_already_recorded(self):
+ with patch.object(tr, "release_translation_quota") as mock_release:
+ await tr._release_quota_if_needed("user-123", usage_recorded=True, job_id="j1")
+ mock_release.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_skips_when_no_user_id(self):
+ with patch.object(tr, "release_translation_quota") as mock_release:
+ await tr._release_quota_if_needed(None, usage_recorded=False, job_id="j1")
+ mock_release.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_swallows_release_errors(self):
+ with patch.object(tr, "release_translation_quota", side_effect=RuntimeError("db down")):
+ # Must not raise even if the release itself fails.
+ await tr._release_quota_if_needed("user-123", usage_recorded=False, job_id="j1")
+
+
+# ---------------------------------------------------------------------------
+# Bug 4 — _compute_duration_seconds (UTC-aware, crash-safe)
+# ---------------------------------------------------------------------------
+class TestComputeDurationSeconds:
+ def test_recent_timestamp_returns_positive(self):
+ ts = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
+ dur = tr._compute_duration_seconds(ts)
+ assert dur >= 9 # allow tiny scheduling slack
+
+ def test_z_suffix_handled(self):
+ ts = (datetime.now(timezone.utc) - timedelta(seconds=5)).strftime("%Y-%m-%dT%H:%M:%SZ")
+ dur = tr._compute_duration_seconds(ts)
+ assert dur >= 0
+
+ def test_malformed_returns_zero(self):
+ # Regression for the original bug: a bad timestamp used to crash the
+ # success path and flip the job to failed. It must now return 0.
+ assert tr._compute_duration_seconds("not-a-date") == 0.0
+
+ def test_empty_returns_zero(self):
+ assert tr._compute_duration_seconds("") == 0.0
+
+
+# ---------------------------------------------------------------------------
+# Bug 5 — _cleanup_old_jobs snapshots the dict (no resize-during-iteration)
+# ---------------------------------------------------------------------------
+class TestCleanupOldJobsSnapshots:
+ def test_cleanup_does_not_raise_when_dict_mutated_concurrently(self, monkeypatch):
+ # Force cleanup to run now (bypass throttle).
+ monkeypatch.setattr(tr, "_last_cleanup_ts", 0.0)
+ monkeypatch.setattr(tr, "_CLEANUP_INTERVAL_SECONDS", 0)
+
+ # Two expired jobs.
+ old_ts = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat()
+ tr._translation_jobs.clear()
+ tr._translation_jobs["j1"] = {"status": "completed", "completed_at": old_ts}
+ tr._translation_jobs["j2"] = {"status": "failed", "failed_at": old_ts}
+ tr._translation_jobs["j3"] = {"status": "running"} # not expired
+
+ # If cleanup did NOT snapshot, mutating during iteration would raise.
+ tr._cleanup_old_jobs()
+
+ assert "j1" not in tr._translation_jobs
+ assert "j2" not in tr._translation_jobs
+ assert "j3" in tr._translation_jobs
+
+ def teardown_method(self):
+ tr._translation_jobs.clear()
diff --git a/tests/test_translation_metadata_integration.py b/tests/test_translation_metadata_integration.py
index 16cc8cb..021d2fb 100644
--- a/tests/test_translation_metadata_integration.py
+++ b/tests/test_translation_metadata_integration.py
@@ -40,7 +40,11 @@ async def test_translate_endpoint_triggers_tracking(client):
with patch(
"routes.translate_routes.storage_tracker.track_file", new_callable=AsyncMock
) as mock_track:
- with patch("routes.translate_routes.file_validator.validate_async") as mock_val:
+ # The upload write is mocked below, so no real archive exists on
+ # disk: neutralize the zip safety check for this test.
+ with patch(
+ "routes.translate_routes.file_validator.validate_async"
+ ) as mock_val, patch("routes.translate_routes.validate_zip_safety"):
mock_val.return_value.is_valid = True
mock_val.return_value.data = {"extension": ".docx", "size_bytes": 500}
@@ -88,7 +92,10 @@ async def test_translate_endpoint_triggers_tracking(client):
async def test_translate_endpoint_handles_hash_failure(client):
app.dependency_overrides[get_authenticated_user] = mock_auth
- with patch("routes.translate_routes.file_validator.validate_async") as mock_val:
+ # No real archive exists on disk (save is mocked): skip the zip check.
+ with patch(
+ "routes.translate_routes.file_validator.validate_async"
+ ) as mock_val, patch("routes.translate_routes.validate_zip_safety"):
mock_val.return_value.is_valid = True
mock_val.return_value.data = {"extension": ".docx", "size_bytes": 500}
diff --git a/tests/test_translators/test_b2_pptx_fixes.py b/tests/test_translators/test_b2_pptx_fixes.py
index 49845d2..f241a5e 100644
--- a/tests/test_translators/test_b2_pptx_fixes.py
+++ b/tests/test_translators/test_b2_pptx_fixes.py
@@ -564,60 +564,46 @@ class TestPptxChartWhitespace:
"""
- def test_padded_chart_text_via_internal_method(self):
- """The internal chart apply logic should preserve whitespace."""
+ def test_chart_translation_reaches_output_file(self, tmp_path):
+ """Chart translations must reach the OUTPUT FILE (real ChartPart).
+
+ python-pptx's ChartPart.blob is read-only — the previous in-memory
+ `blob = ...` write never landed in the saved .pptx. This end-to-end
+ test uses a real chart and verifies the chart XML inside the
+ output ZIP.
+ """
+ from pptx.chart.data import CategoryChartData
+ from pptx.enum.chart import XL_CHART_TYPE
+
provider = MockProvider({"Padded chart title": "Titre avec espaces"})
translator = PowerPointTranslator(provider=provider)
- # Build a chart entry by hand (simulating collect time)
- chart_xml = etree.fromstring(self.CHART_PADDED_XML.encode("utf-8"))
- entries = []
- for t_elem in chart_xml.iter(f"{{{_NS_A}}}t"):
- text_raw = t_elem.text or ""
- text = text_raw.strip()
- if not text:
- continue
- entry = {
- "element": t_elem,
- "original": text,
- "original_raw": text_raw,
- "translated": "Titre avec espaces",
- "tag": "a:t",
- "element_path": translator._get_element_path(t_elem),
- }
- entries.append(entry)
-
- if not hasattr(translator, "_chart_entries"):
- translator._chart_entries = []
-
- class _FakePart:
- def __init__(self, blob):
- self._blob = blob
- @property
- def blob(self):
- return self._blob
- @blob.setter
- def blob(self, value):
- self._blob = value
-
- fake_part = _FakePart(
- etree.tostring(
- chart_xml, xml_declaration=True, encoding="UTF-8", standalone=True
- )
+ prs = Presentation()
+ slide = prs.slides.add_slide(prs.slide_layouts[5])
+ chart_data = CategoryChartData()
+ chart_data.categories = ["A", "B"]
+ chart_data.add_series("Series 1", (1, 2))
+ graphic_frame = slide.shapes.add_chart(
+ XL_CHART_TYPE.COLUMN_CLUSTERED, 10, 10, 400, 300, chart_data
)
- translator._chart_entries.append({
- "chart_part": fake_part,
- "entries": entries,
- })
+ chart = graphic_frame.chart
+ chart.has_title = True
+ chart.chart_title.text_frame.text = "Padded chart title"
- # Apply
- translator._apply_chart_translations(Path("dummy"))
+ input_file = tmp_path / "chart_in.pptx"
+ output_file = tmp_path / "chart_out.pptx"
+ prs.save(str(input_file))
- # Re-parse and check whitespace preserved
- updated_xml = etree.fromstring(fake_part._blob)
- all_t = list(updated_xml.iter(f"{{{_NS_A}}}t"))
- # Find the title text
- title_text = all_t[0].text or ""
- assert " Titre avec espaces " in title_text, (
- f"Chart whitespace not preserved: {title_text!r}"
+ translator.translate_file(input_file, output_file, "fr", "en")
+
+ with zipfile.ZipFile(output_file, "r") as zf:
+ chart_parts = [
+ n for n in zf.namelist() if n.startswith("ppt/charts/chart")
+ ]
+ assert chart_parts, "chart part missing from output file"
+ chart_xml = zf.read(chart_parts[0]).decode("utf-8")
+
+ assert "Titre avec espaces" in chart_xml, (
+ "Chart title translation never reached the output file"
)
+ assert "Padded chart title" not in chart_xml
diff --git a/tests/test_translators/test_pdf_quality_fixes.py b/tests/test_translators/test_pdf_quality_fixes.py
new file mode 100644
index 0000000..b0da52c
--- /dev/null
+++ b/tests/test_translators/test_pdf_quality_fixes.py
@@ -0,0 +1,139 @@
+"""PDF quality fixes: table-cell merge guard, bold/italic fonts, unchanged
+blocks left untouched (no redaction/rewrite), and stats propagation."""
+
+import fitz
+import pytest
+
+from translators.pdf_translator import PDFTranslator
+
+
+class TestTableCellMergeGuard:
+ def test_table_cells_never_merge(self):
+ t = PDFTranslator(provider=None)
+ a = {
+ "bbox": (72, 100, 200, 115),
+ "font_size": 11,
+ "_is_table_cell": True,
+ }
+ b = {
+ "bbox": (72, 118, 200, 133), # same column, next row
+ "font_size": 11,
+ "_is_table_cell": True,
+ }
+ assert t._should_merge_blocks(a, b) is False
+
+ def test_table_cell_and_paragraph_do_not_merge(self):
+ t = PDFTranslator(provider=None)
+ a = {"bbox": (72, 100, 200, 115), "font_size": 11, "_is_table_cell": True}
+ b = {"bbox": (72, 118, 200, 133), "font_size": 11}
+ assert t._should_merge_blocks(a, b) is False
+
+ def test_regular_paragraphs_still_merge(self):
+ t = PDFTranslator(provider=None)
+ a = {"bbox": (72, 100, 300, 115), "font_size": 11}
+ b = {"bbox": (72, 118, 302, 133), "font_size": 11}
+ assert t._should_merge_blocks(a, b) is True
+
+
+class TestBoldItalicFontSelection:
+ def _capturing_page(self):
+ page = fitz.open().new_page()
+ calls = []
+ original = page.insert_textbox
+
+ def capture(rect, text, fontname=None, fontfile=None, fontsize=None, **kw):
+ calls.append({"fontname": fontname, "fontfile": fontfile})
+ return 0
+
+ page.insert_textbox = capture
+ return page, calls
+
+ def _block(self, **kwargs):
+ block = {
+ "bbox": (72, 100, 400, 130),
+ "text": "Bold heading",
+ "translated": "Titre en gras",
+ "font_size": 20,
+ "color": 0,
+ "line_count": 1,
+ "sub_bboxes": [(72, 100, 400, 130)],
+ }
+ block.update(kwargs)
+ return block
+
+ def test_bold_block_uses_hebo(self):
+ page, calls = self._capturing_page()
+ t = PDFTranslator(provider=None)
+ t._font_path = None # force base-14 selection
+ t._write_translated_block(page, self._block(is_bold=True), None, False)
+ assert calls and calls[0]["fontname"] == "hebo"
+
+ def test_italic_block_uses_heit(self):
+ page, calls = self._capturing_page()
+ t = PDFTranslator(provider=None)
+ t._font_path = None
+ t._write_translated_block(page, self._block(is_italic=True), None, False)
+ assert calls and calls[0]["fontname"] == "heit"
+
+ def test_bold_italic_uses_hebi(self):
+ page, calls = self._capturing_page()
+ t = PDFTranslator(provider=None)
+ t._font_path = None
+ t._write_translated_block(
+ page, self._block(is_bold=True, is_italic=True), None, False
+ )
+ assert calls and calls[0]["fontname"] == "hebi"
+
+ def test_regular_block_keeps_helv(self):
+ page, calls = self._capturing_page()
+ t = PDFTranslator(provider=None)
+ t._font_path = None
+ t._write_translated_block(page, self._block(), None, False)
+ assert calls and calls[0]["fontname"] == "helv"
+
+
+class _IdentityProvider:
+ """Returns the input text unchanged (new-style provider)."""
+
+ def get_name(self):
+ return "identity"
+
+ def is_available(self):
+ return True
+
+ def translate_text(self, request):
+ from services.providers.schemas import TranslationResponse
+
+ return TranslationResponse(
+ translated_text=request.text,
+ provider_name="identity",
+ from_cache=False,
+ )
+
+
+class TestUnchangedBlocksUntouched:
+ def test_identity_translation_stats_and_no_rewrite(self, tmp_path):
+ """Provider returning the source text: stats stay changed=0 (so the
+ route gate detects it) and blocks are left un-redacted."""
+ doc = fitz.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Already in English, nothing to do.", fontsize=11)
+ src = tmp_path / "en.pdf"
+ doc.save(str(src))
+ doc.close()
+
+ t = PDFTranslator(provider=_IdentityProvider())
+ out = tmp_path / "out.pdf"
+ result = t.translate_file(src, out, "en", "auto")
+
+ assert result.exists()
+ stats = t.get_translation_stats()
+ assert stats["attempted"] >= 1
+ assert stats["changed"] == 0
+
+ # The text is still there, byte-for-byte same rendering (block was
+ # not redacted + rewritten in the substitute font).
+ check = fitz.open(str(result))
+ text = check[0].get_text("text")
+ check.close()
+ assert "Already in English" in text
diff --git a/tests/test_translators/test_word_translator.py b/tests/test_translators/test_word_translator.py
index 3657391..2316f6b 100644
--- a/tests/test_translators/test_word_translator.py
+++ b/tests/test_translators/test_word_translator.py
@@ -190,20 +190,23 @@ class TestParagraphTranslation:
"""Tests for paragraph text translation (AC1)."""
def test_translate_paragraph_runs(self, tmp_path):
- """Test that paragraph runs are translated."""
+ """Adjacent runs with identical formatting merge into ONE unit.
+
+ "Hello" + " " + "World" (same formatting, rsid-style splits) must be
+ translated as the whole sentence, not as separate fragments.
+ """
mock_provider = MockTranslationProvider(
{
- "Hello": "Bonjour",
- "World": "Monde",
+ "Hello World": "Bonjour le monde",
}
)
translator = WordTranslator(provider=mock_provider)
doc = Document()
para = doc.add_paragraph()
- run1 = para.add_run("Hello")
- run2 = para.add_run(" ")
- run3 = para.add_run("World")
+ para.add_run("Hello")
+ para.add_run(" ")
+ para.add_run("World")
input_file = tmp_path / "input.docx"
output_file = tmp_path / "output.docx"
@@ -214,8 +217,41 @@ class TestParagraphTranslation:
doc_out = Document(output_file)
text = doc_out.paragraphs[0].text
- assert "Bonjour" in text
- assert "Monde" in text
+ assert text == "Bonjour le monde"
+ # One merged unit → a single provider call
+ assert mock_provider._call_count == 1
+
+ def test_bold_span_kept_separate_and_coherent(self, tmp_path):
+ """A formatting change mid-sentence splits the units; spaces survive."""
+ mock_provider = MockTranslationProvider(
+ {
+ "This is": "Ceci est",
+ "very important": "très important",
+ }
+ )
+ translator = WordTranslator(provider=mock_provider)
+
+ doc = Document()
+ para = doc.add_paragraph()
+ para.add_run("This is ")
+ bold = para.add_run("very important")
+ bold.bold = True
+
+ input_file = tmp_path / "input.docx"
+ output_file = tmp_path / "output.docx"
+ doc.save(input_file)
+
+ translator.translate_file(input_file, output_file, "fr")
+
+ doc_out = Document(output_file)
+ text = doc_out.paragraphs[0].text
+ assert text == "Ceci est très important"
+
+ # Bold formatting survives on the right span
+ runs = [r for r in doc_out.paragraphs[0].runs if r.text.strip()]
+ assert len(runs) == 2
+ assert runs[1].bold is True
+ assert runs[1].text == "très important"
def test_empty_paragraphs_not_translated(self, tmp_path):
"""Test that empty paragraphs are not translated."""
diff --git a/translations.db b/translations.db
deleted file mode 100644
index e69de29..0000000
diff --git a/translators/bilingual.py b/translators/bilingual.py
new file mode 100644
index 0000000..b767fd8
--- /dev/null
+++ b/translators/bilingual.py
@@ -0,0 +1,90 @@
+"""
+Bilingual output: interleave source paragraphs with their translation.
+
+Given the ORIGINAL document and the TRANSLATED document (same structure —
+the pipeline only rewrites run texts), produce a copy of the translated
+document where each translated body paragraph is preceded by its source
+paragraph in gray italic. Tables, headers and footers keep the translated
+version only (duplicating them would double the layout).
+"""
+
+from pathlib import Path
+from typing import Optional
+
+from docx import Document
+from docx.text.paragraph import Paragraph
+from docx.oxml import OxmlElement
+from docx.oxml.ns import qn
+from docx.shared import Pt, RGBColor
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def make_bilingual_docx(
+ source_path: Path, translated_path: Path, output_path: Path
+) -> Optional[Path]:
+ """Create a bilingual .docx (source paragraph above its translation).
+
+ Returns the output path, or None when the pairing failed (structure
+ mismatch) — callers fall back to the translated-only file.
+ """
+ source_path = Path(source_path)
+ translated_path = Path(translated_path)
+ output_path = Path(output_path)
+
+ try:
+ src = Document(str(source_path))
+ tr = Document(str(translated_path))
+ except Exception as e:
+ logger.warning("bilingual_open_failed", error=str(e))
+ return None
+
+ src_children = list(src.element.body)
+ tr_children = list(tr.element.body)
+
+ # The pipeline preserves structure exactly; a drift beyond a small
+ # tolerance means pairing by index is unsafe → bail out.
+ if abs(len(src_children) - len(tr_children)) > 0:
+ logger.warning(
+ "bilingual_structure_mismatch",
+ source_elements=len(src_children),
+ translated_elements=len(tr_children),
+ )
+ if len(src_children) != len(tr_children):
+ return None
+
+ inserted = 0
+ for src_el, tr_el in zip(src_children, tr_children):
+ if tr_el.tag != qn("w:p") or src_el.tag != qn("w:p"):
+ continue
+ src_text = "".join(
+ t.text or "" for t in src_el.iter(qn("w:t"))
+ ).strip()
+ if not src_text:
+ continue
+
+ # Insert a NEW paragraph directly above the translated one, inside
+ # the translated document (keeps styles/sections untouched).
+ new_p = OxmlElement("w:p")
+ tr_el.addprevious(new_p)
+ para = Paragraph(new_p, tr)
+ run = para.add_run(src_text)
+ run.font.size = Pt(9)
+ run.font.italic = True
+ run.font.color.rgb = RGBColor(0x80, 0x80, 0x80)
+ inserted += 1
+
+ if inserted == 0:
+ logger.info("bilingual_nothing_inserted")
+ return None
+
+ try:
+ tr.save(str(output_path))
+ except Exception as e:
+ logger.warning("bilingual_save_failed", error=str(e))
+ return None
+
+ logger.info("bilingual_docx_created", paragraphs=inserted)
+ return output_path
diff --git a/translators/excel_translator.py b/translators/excel_translator.py
index dee144d..b2b4b59 100644
--- a/translators/excel_translator.py
+++ b/translators/excel_translator.py
@@ -107,6 +107,7 @@ class ExcelTranslator:
self._provider = provider
self.formula_pattern = re.compile(r"=.*")
self._custom_prompt: Optional[str] = None
+ self._tm_scope = None # set via set_tm_scope (per-user translation memory)
self._translation_stats = {"attempted": 0, "changed": 0}
def set_provider(self, provider: TranslationProvider) -> None:
@@ -116,6 +117,14 @@ class ExcelTranslator:
def set_custom_prompt(self, prompt: Optional[str]) -> None:
"""Set custom system prompt for LLM providers."""
self._custom_prompt = prompt
+ def set_tm_scope(self, user_id, prompt=None) -> None:
+ """Enable the per-user translation memory for this job."""
+ from services.translation_tm import TMScope
+
+ self._tm_scope = TMScope.from_prompt(
+ user_id, prompt or getattr(self, "_custom_prompt", None)
+ )
+
def translate_file(
self,
@@ -318,6 +327,13 @@ class ExcelTranslator:
new_name=new_name,
)
+ # openpyxl does NOT rewrite references on rename: cell
+ # formulas, defined names and chart refs pointing at the old
+ # sheet would break (#REF!/#NAME?). Charts are handled later
+ # via ZIP re-injection; fix cells + defined names here.
+ if sheet_name_mapping:
+ self._rewrite_sheet_refs_in_workbook(workbook, sheet_name_mapping)
+
if translate_images:
_log_info("excel_image_translation_start", sheets=len(workbook.sheetnames))
for sheet_name in workbook.sheetnames:
@@ -427,12 +443,30 @@ class ExcelTranslator:
non_empty = [t for t in texts if t and t.strip()]
self._translation_stats["attempted"] += len(non_empty)
+ from services.translation_tm import translate_with_tm
+
+ provider_name = (
+ self._provider.get_name() if hasattr(self._provider, "get_name")
+ else type(self._provider).__name__
+ ) if self._provider is not None else "legacy"
+
if self._provider is not None:
- translated = self._translate_with_provider(
- texts, target_language, source_language
- )
+ def _do_translate(miss_texts):
+ return self._translate_with_provider(
+ miss_texts, target_language, source_language
+ )
else:
- translated = self._translate_with_legacy(texts, target_language, source_language)
+ def _do_translate(miss_texts):
+ return self._translate_with_legacy(
+ miss_texts, target_language, source_language
+ )
+
+ # Translation memory: reuse this user's previous translations
+ # (identical context/prompt) before hitting the provider.
+ translated = translate_with_tm(
+ texts, target_language, source_language,
+ provider_name, getattr(self, "_tm_scope", None), _do_translate,
+ )
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
self._translation_stats["changed"] += changed
@@ -931,6 +965,140 @@ class ExcelTranslator:
rewritten += 1
return rewritten
+ @staticmethod
+ def _rewrite_sheet_refs_in_formula(
+ formula: str, sheet_name_mapping: Dict[str, str]
+ ) -> str:
+ """Rewrite every sheet reference inside a formula string.
+
+ Handles quoted ('Ventes 2026'!), unquoted (Ventes!) and 3D refs
+ (Sheet1!A1:Sheet2!B2 — each end is rewritten independently).
+ Longest names are replaced first so a name that is a prefix of
+ another is not corrupted.
+ """
+ if not formula or not sheet_name_mapping or "!" not in formula:
+ return formula
+
+ result = formula
+ # longest first to avoid partial-name collisions
+ for old in sorted(sheet_name_mapping, key=len, reverse=True):
+ new = sheet_name_mapping[old]
+ if new == old:
+ continue
+ new_quoted = ExcelTranslator._quote_sheet_name_for_ref(new)
+ # Quoted form: inner apostrophes are escaped as ''
+ escaped = old.replace("'", "''")
+ result = re.sub(
+ rf"'{re.escape(escaped)}'!",
+ new_quoted + "!",
+ result,
+ )
+ # Unquoted form: only when the old name needs no quotes; guard
+ # against matching the tail of a longer name.
+ if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.]*", old):
+ result = re.sub(
+ rf"(? None:
+ """After a sheet rename, fix every reference that still points at the
+ old names: cell formulas, defined names, data validations and
+ conditional-formatting rules. openpyxl does none of this on rename.
+
+ Best-effort: individual failures are logged and skipped — a broken
+ rename must never lose the whole file.
+ """
+ if not sheet_name_mapping:
+ return
+
+ cells_fixed = names_fixed = validations_fixed = cf_fixed = 0
+
+ try:
+ for worksheet in workbook.worksheets:
+ # 1) Cell formulas
+ for row in worksheet.iter_rows():
+ for cell in row:
+ value = cell.value
+ if isinstance(value, str) and value.startswith("=") and "!" in value:
+ updated = cls._rewrite_sheet_refs_in_formula(
+ value, sheet_name_mapping
+ )
+ if updated != value:
+ cell.value = updated
+ cells_fixed += 1
+
+ # 2) Data validations (list sources / custom formulas)
+ try:
+ for dv in worksheet.data_validations.dataValidation:
+ for attr in ("formula1", "formula2"):
+ fx = getattr(dv, attr, None)
+ if isinstance(fx, str) and "!" in fx:
+ updated = cls._rewrite_sheet_refs_in_formula(
+ fx, sheet_name_mapping
+ )
+ if updated != fx:
+ setattr(dv, attr, updated)
+ validations_fixed += 1
+ except Exception as e:
+ _log_warning(
+ "excel_sheet_refs_validations_failed",
+ sheet=worksheet.title,
+ error=str(e),
+ )
+
+ # 3) Conditional formatting rules
+ try:
+ for cf in worksheet.conditional_formatting:
+ for rule in cf.rules:
+ formulas = getattr(rule, "formula", None) or []
+ for idx, fx in enumerate(formulas):
+ if isinstance(fx, str) and "!" in fx:
+ updated = cls._rewrite_sheet_refs_in_formula(
+ fx, sheet_name_mapping
+ )
+ if updated != fx:
+ formulas[idx] = updated
+ cf_fixed += 1
+ except Exception as e:
+ _log_warning(
+ "excel_sheet_refs_condfmt_failed",
+ sheet=worksheet.title,
+ error=str(e),
+ )
+
+ # 4) Workbook-level defined names
+ try:
+ for name in list(workbook.defined_names.values()):
+ attr_text = getattr(name, "attr_text", None)
+ if isinstance(attr_text, str) and "!" in attr_text:
+ updated = cls._rewrite_sheet_refs_in_formula(
+ attr_text, sheet_name_mapping
+ )
+ if updated != attr_text:
+ name.attr_text = updated
+ names_fixed += 1
+ except Exception as e:
+ _log_warning("excel_sheet_refs_names_failed", error=str(e))
+
+ except Exception as e:
+ _log_error("excel_sheet_refs_rewrite_failed", error=str(e))
+ return
+
+ if cells_fixed or names_fixed or validations_fixed or cf_fixed:
+ _log_info(
+ "excel_sheet_refs_rewritten",
+ cells=cells_fixed,
+ defined_names=names_fixed,
+ validations=validations_fixed,
+ conditional_formats=cf_fixed,
+ )
+
def _translate_images(self, worksheet: Worksheet, target_language: str) -> None:
"""
Translate text in images using vision model.
diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py
index 4aa9742..97d32ac 100644
--- a/translators/pdf_translator.py
+++ b/translators/pdf_translator.py
@@ -21,6 +21,13 @@ Fallback:
Text-only mode:
Extract text, translate, generate a clean formatted PDF via reportlab.
+
+Scanned PDFs:
+ Image-only PDFs have no text layer to extract or rewrite. They are
+ detected up front (average extractable characters per page below
+ config.SCANNED_PDF_MIN_CHARS_PER_PAGE) and routed through the Mistral
+ OCR API (services/mistral_ocr.py) before translation; the output is a
+ clean re-typeset PDF (layout is not preserved — the source is images).
"""
import time
@@ -114,8 +121,15 @@ class PDFTranslator:
"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
"/app/fonts/NotoSans-Regular.ttf",
+ # CJK-capable fonts (target languages zh/ja/ko render as tofu with
+ # a Latin-only font file)
+ "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
+ "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc",
+ "/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf",
"C:/Windows/Fonts/arial.ttf",
"C:/Windows/Fonts/msyh.ttc",
+ "C:/Windows/Fonts/simsun.ttc",
+ "C:/Windows/Fonts/msgothic.ttc",
"/System/Library/Fonts/Helvetica.ttc",
]
@@ -124,6 +138,11 @@ class PDFTranslator:
self._font_path: Optional[str] = None
self._translation_stats = {"attempted": 0, "changed": 0}
self._custom_prompt: Optional[str] = None
+ # OCR overrides (admin settings); None → fall back to config.MISTRAL_*
+ self._ocr_api_key: Optional[str] = None
+ self._ocr_model: Optional[str] = None
+ self._ocr_timeout: Optional[int] = None
+ self._ocr_enabled: Optional[bool] = None
def set_provider(self, provider) -> None:
"""Set the translation provider."""
@@ -133,6 +152,26 @@ class PDFTranslator:
"""Set custom system prompt for LLM providers."""
self._custom_prompt = prompt
+ def set_ocr_config(
+ self,
+ api_key: Optional[str] = None,
+ model: Optional[str] = None,
+ timeout: Optional[int] = None,
+ enabled: Optional[bool] = None,
+ ) -> None:
+ """Configure the Mistral OCR call (admin settings > env defaults).
+
+ Only the values explicitly provided override the config defaults,
+ so the route can pass admin-configured values and let the rest fall
+ back to ``config.MISTRAL_*``.
+ """
+ self._ocr_api_key = api_key
+ self._ocr_model = model
+ if timeout is not None:
+ self._ocr_timeout = timeout
+ if enabled is not None:
+ self._ocr_enabled = enabled
+
def _get_font_path(self) -> Optional[str]:
"""Resolve a Unicode-capable TTF/OTF font file."""
if self._font_path is not None:
@@ -159,6 +198,14 @@ class PDFTranslator:
output_path = Path(output_path)
self._validate_file(input_path)
+ # Scanned PDFs must be detected before either mode: both rely on an
+ # extractable text layer, which image-only pages don't have.
+ if self._is_scanned_pdf(input_path):
+ return self._translate_scanned_pdf(
+ input_path, output_path, target_language, source_language,
+ progress_callback,
+ )
+
if pdf_mode == "text_only":
return self._translate_text_only(
input_path, output_path, target_language, source_language, progress_callback
@@ -294,8 +341,22 @@ class PDFTranslator:
original, target_language, source_language
)
if translated and translated.strip():
- block["translated"] = translated
- translated_blocks += 1
+ if translated.strip() == original.strip():
+ # Unchanged (already in the target language, or
+ # the provider failed and returned the source
+ # text). Leave the block completely untouched —
+ # redacting and rewriting identical text would
+ # only degrade its typography (embedded fonts
+ # lost, everything redrawn in the substitute).
+ block["translated"] = None
+ logger.info(
+ "block_translation_unchanged",
+ page=page_num + 1,
+ text_preview=original[:60],
+ )
+ else:
+ block["translated"] = translated
+ translated_blocks += 1
else:
logger.warning(
"block_translation_empty",
@@ -357,6 +418,7 @@ class PDFTranslator:
return True
return False
+ redacted_rects: list = []
for block in blocks:
if block.get("translated"):
# Track B3.5: single redaction per block, not per sub-bbox.
@@ -371,6 +433,7 @@ class PDFTranslator:
page.add_redact_annot(block_bbox, fill=None)
else:
page.add_redact_annot(block_bbox, fill=(1, 1, 1))
+ redacted_rects.append(block_bbox)
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
@@ -378,7 +441,11 @@ class PDFTranslator:
# We use the original link geometry (it's unaffected by the
# redaction). URIs are preserved verbatim; only the visible
# text changes.
- if page_links_before:
+ # Only links intersecting an actually-redacted block are
+ # re-inserted: when a page has no redaction at all (every block
+ # unchanged), the original annotations survive apply_redactions
+ # and re-inserting would DUPLICATE them.
+ if page_links_before and redacted_rects:
reinserted = 0
lost = 0
page_rect = page.rect
@@ -393,6 +460,12 @@ class PDFTranslator:
if not page_rect.intersects(from_rect):
lost += 1
continue
+ # Skip links outside every redacted area — their
+ # original annotation is still alive on the page.
+ if not any(
+ from_rect.intersects(r) for r in redacted_rects
+ ):
+ continue
# Build the link insertion kwargs based on kind
if link.get("uri"):
# External URI link
@@ -700,6 +773,13 @@ class PDFTranslator:
if a.get("_no_merge") or b.get("_no_merge"):
return False
+ # Table cells: consecutive rows of the same column satisfy every
+ # geometric merge condition (same x0, similar width, small positive
+ # gap) but are UNRELATED values — merging them joins two cells into
+ # one paragraph spanning both rows and breaks the table structure.
+ if a.get("_is_table_cell") or b.get("_is_table_cell"):
+ return False
+
# Must have similar font size (within 20%)
if abs(a["font_size"] - b["font_size"]) > max(a["font_size"], b["font_size"]) * 0.2:
return False
@@ -812,8 +892,22 @@ class PDFTranslator:
# PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'.
# If a custom font file is available, use it via fontfile (fontname ignored).
- fontname = "helv"
- fontfile = font_path
+ # Base-14 font variant honouring the block's bold/italic flags —
+ # previously every block (headings included) rendered in regular.
+ # When a custom Unicode fontfile is used we keep it: we have no
+ # bold/italic variant of that file, and glyph coverage matters more
+ # than weight.
+ if font_path:
+ fontname = "helv"
+ fontfile = font_path
+ else:
+ fontname = (
+ "hebi" if (block.get("is_bold") and block.get("is_italic"))
+ else "hebo" if block.get("is_bold")
+ else "heit" if block.get("is_italic")
+ else "helv"
+ )
+ fontfile = None
# Determine if this is a heading (larger font size = more visual weight)
is_heading = target_size >= HEADING_MIN_SIZE
@@ -998,6 +1092,12 @@ class PDFTranslator:
progress_callback=None,
translate_images=translate_images,
)
+ # Propagate the inner Word stats so the route's sanity gate
+ # (attempted/changed) works on the fallback path too.
+ for key, value in wt.get_translation_stats().items():
+ self._translation_stats[key] = (
+ self._translation_stats.get(key, 0) + value
+ )
if progress_callback:
progress_callback({
@@ -1125,6 +1225,31 @@ class PDFTranslator:
pages_text.append(text)
doc.close()
+ translated_pages = self._translate_page_texts(
+ pages_text, target_language, source_language, progress_callback
+ )
+
+ final_path = output_path.with_suffix(".pdf")
+ self._generate_clean_pdf(translated_pages, final_path, target_language)
+
+ processing_time_ms = round((time.time() - start_time) * 1000, 2)
+ logger.info(
+ "pdf_text_only_success",
+ file_name=input_path.name,
+ pages=total_pages,
+ processing_time_ms=processing_time_ms,
+ )
+
+ return final_path
+
+ def _translate_page_texts(
+ self,
+ pages_text: List[str],
+ target_language: str,
+ source_language: str,
+ progress_callback,
+ ) -> List[str]:
+ """Translate per-page texts, keeping order; empty pages pass through."""
non_empty_indices = [i for i, t in enumerate(pages_text) if t]
if progress_callback:
@@ -1135,6 +1260,7 @@ class PDFTranslator:
})
translated_pages = list(pages_text)
+ total_pages = len(pages_text)
for seq, page_idx in enumerate(non_empty_indices):
text = pages_text[page_idx]
@@ -1160,19 +1286,139 @@ class PDFTranslator:
"progress_override": pct,
})
+ return translated_pages
+
+ # ------------------------------------------------------------------ #
+ # SCANNED PDFs — Mistral OCR
+ # ------------------------------------------------------------------ #
+
+ def _is_scanned_pdf(self, input_path: Path) -> bool:
+ """True when the PDF is image-only (no usable text layer).
+
+ A page counts as a scan page when it has almost no extractable
+ text AND is mostly covered by raster images. The document is
+ considered scanned when it contains scan pages and no page with a
+ real text layer — this keeps sparse-but-textual PDFs (a bare
+ title page, a single label) on the normal layout pipeline.
+ """
+ from config import config
+
+ try:
+ import fitz
+ except ImportError:
+ return False
+
+ try:
+ with fitz.open(str(input_path)) as doc:
+ if len(doc) == 0:
+ return False
+ has_text_page = False
+ has_scan_page = False
+ for page in doc:
+ if len(page.get_text("text").strip()) >= config.SCANNED_PDF_MIN_CHARS_PER_PAGE:
+ has_text_page = True
+ continue
+ # Text-poor page: a scan page only when raster images
+ # cover most of its area (a bare title page has none).
+ page_area = abs(page.rect)
+ img_area = 0.0
+ for img in page.get_images(full=True):
+ for rect in page.get_image_rects(img[0]):
+ img_area += abs(rect & page.rect)
+ if page_area and img_area / page_area >= 0.5:
+ has_scan_page = True
+ except Exception as e:
+ logger.warning("scanned_pdf_detection_failed", error=str(e))
+ return False
+
+ scanned = has_scan_page and not has_text_page
+ if scanned:
+ logger.info(
+ "scanned_pdf_detected",
+ file=input_path.name,
+ )
+ return scanned
+
+ def _translate_scanned_pdf(
+ self,
+ input_path: Path,
+ output_path: Path,
+ target_language: str,
+ source_language: str,
+ progress_callback,
+ ) -> Path:
+ """OCR (Mistral) → translate → clean re-typeset PDF.
+
+ The source pages are images, so the original layout cannot be
+ rewritten in place; the output carries the recovered text in a
+ clean document instead.
+ """
+ from config import config
+ from services.mistral_ocr import MistralOCRClient, MistralOCRError
+
+ # Resolution order: admin settings (via set_ocr_config) > env/config.
+ api_key = (self._ocr_api_key or "").strip() or config.MISTRAL_API_KEY
+ model = (self._ocr_model or "").strip() or config.MISTRAL_OCR_MODEL
+ timeout = self._ocr_timeout or config.MISTRAL_OCR_TIMEOUT
+ ocr_enabled = (
+ config.MISTRAL_OCR_ENABLED
+ if self._ocr_enabled is None
+ else self._ocr_enabled
+ )
+
+ if not ocr_enabled or not api_key:
+ raise RuntimeError(
+ "PDF scanné détecté (pages image sans couche texte). "
+ "La traduction des PDF scannés nécessite l'OCR Mistral : "
+ "configurez MISTRAL_API_KEY (ou fournissez un PDF avec du texte sélectionnable)."
+ )
+
+ start_time = time.time()
+ client = MistralOCRClient(
+ api_key=api_key,
+ model=model,
+ timeout=timeout,
+ )
+
+ pages_markdown = client.extract_pdf_text(
+ input_path, progress_callback=progress_callback
+ )
+ pages_text = [self._markdown_to_text(md) for md in pages_markdown]
+
+ translated_pages = self._translate_page_texts(
+ pages_text, target_language, source_language, progress_callback
+ )
+
final_path = output_path.with_suffix(".pdf")
self._generate_clean_pdf(translated_pages, final_path, target_language)
processing_time_ms = round((time.time() - start_time) * 1000, 2)
logger.info(
- "pdf_text_only_success",
+ "pdf_scanned_success",
file_name=input_path.name,
- pages=total_pages,
+ pages=len(pages_text),
processing_time_ms=processing_time_ms,
)
-
return final_path
+ @staticmethod
+ def _markdown_to_text(markdown: str) -> str:
+ """Flatten OCR markdown to plain text (drop images/links/markup)."""
+ import re
+
+ if not markdown:
+ return ""
+ text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", markdown) # images
+ text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) # links → label
+ text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) # headings
+ text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) # bullets
+ # Markdown table rows → plain line of cells
+ text = re.sub(r"^\s*\|", "", text, flags=re.MULTILINE)
+ text = text.replace("|", " ")
+ text = re.sub(r"^\s*[-:| ]+\s*$", "", text, flags=re.MULTILINE) # rules
+ text = re.sub(r"\n{3,}", "\n\n", text)
+ return text.strip()
+
def _generate_clean_pdf(
self, pages_text: List[str], output_path: Path, target_language: str = "en"
) -> None:
@@ -1305,18 +1551,29 @@ class PDFTranslator:
def _translate_single(
self, text: str, target_language: str, source_language: str
) -> str:
- """Translate a single text string."""
+ """Translate a single text string.
+
+ Also feeds the job-level attempted/changed stats so the route can
+ detect a total provider failure (changed == 0) on PDFs too.
+ """
+ if text and text.strip():
+ self._translation_stats["attempted"] += 1
if self._provider is not None:
try:
results = self._translate_with_provider([text], target_language, source_language)
if results and results[0].strip():
+ if results[0].strip() != text.strip():
+ self._translation_stats["changed"] += 1
return results[0]
except Exception as e:
logger.warning("provider_single_failed", error=str(e))
from services.translation_service import translation_service
try:
- return translation_service.translate_text(text, target_language, source_language)
+ result = translation_service.translate_text(text, target_language, source_language)
+ if result and result.strip() and result.strip() != text.strip():
+ self._translation_stats["changed"] += 1
+ return result
except Exception as e:
logger.warning("legacy_single_failed", error=str(e))
return text
diff --git a/translators/pptx_translator.py b/translators/pptx_translator.py
index 0b372e5..efcedb0 100644
--- a/translators/pptx_translator.py
+++ b/translators/pptx_translator.py
@@ -95,6 +95,69 @@ def _apply_rtl_to_shape(shape) -> None:
_apply_rtl_to_shape(sub_shape)
+# East-Asian typeface hints per target language (DrawingML )
+_EA_TYPEFACES = {
+ "zh": "SimSun",
+ "zh-CN": "SimSun",
+ "zh-TW": "PMingLiU",
+ "ja": "Yu Mincho",
+ "ko": "Batang",
+}
+
+
+def _ea_typeface_for_target(target_language: str):
+ code = (target_language or "").strip()
+ base = code.split("-")[0].lower()
+ return _EA_TYPEFACES.get(code) or _EA_TYPEFACES.get(base)
+
+
+def _apply_ea_font_hints(presentation: Presentation, target_language: str) -> None:
+ """Set the (east-asian) typeface on every run for CJK targets.
+
+ Blanket application is safe — the hint only affects CJK glyphs.
+ """
+
+ def _hint_shape(shape) -> int:
+ hinted = 0
+ if shape.has_text_frame:
+ hinted += _hint_text_frame(shape.text_frame)
+ if shape.shape_type == MSO_SHAPE_TYPE.TABLE:
+ for row in shape.table.rows:
+ for cell in row.cells:
+ hinted += _hint_text_frame(cell.text_frame)
+ if shape.shape_type == MSO_SHAPE_TYPE.GROUP:
+ for sub_shape in shape.shapes:
+ hinted += _hint_shape(sub_shape)
+ return hinted
+
+ def _hint_text_frame(text_frame) -> int:
+ hinted = 0
+ tag_rPr = f"{{{_NS_A}}}rPr"
+ tag_ea = f"{{{_NS_A}}}ea"
+ for paragraph in text_frame.paragraphs:
+ for run in paragraph.runs:
+ rPr = run._r.find(tag_rPr)
+ if rPr is None:
+ rPr = etree.SubElement(run._r, tag_rPr)
+ ea = rPr.find(tag_ea)
+ if ea is None:
+ ea = etree.SubElement(rPr, tag_ea)
+ if not ea.get("typeface"):
+ ea.set("typeface", typeface)
+ hinted += 1
+ return hinted
+
+ typeface = _ea_typeface_for_target(target_language)
+ if not typeface:
+ return
+ total = 0
+ for slide in presentation.slides:
+ for shape in slide.shapes:
+ total += _hint_shape(shape)
+ if total:
+ _log_info("pptx_ea_font_hints_applied", runs=total, typeface=typeface)
+
+
class PptxProcessorError(Exception):
"""Exception for PowerPoint processing errors with structured error codes."""
@@ -152,6 +215,7 @@ class PowerPointTranslator:
"""
self._provider = provider
self._custom_prompt: Optional[str] = None
+ self._tm_scope = None # set via set_tm_scope (per-user translation memory)
self._translation_stats = {"attempted": 0, "changed": 0}
def set_provider(self, provider: TranslationProvider) -> None:
@@ -161,6 +225,14 @@ class PowerPointTranslator:
def set_custom_prompt(self, prompt: Optional[str]) -> None:
"""Set custom system prompt for LLM providers."""
self._custom_prompt = prompt
+ def set_tm_scope(self, user_id, prompt=None) -> None:
+ """Enable the per-user translation memory for this job."""
+ from services.translation_tm import TMScope
+
+ self._tm_scope = TMScope.from_prompt(
+ user_id, prompt or getattr(self, "_custom_prompt", None)
+ )
+
def translate_file(
self,
@@ -316,6 +388,10 @@ class PowerPointTranslator:
if target_language.lower() in RTL_LANGUAGES:
_apply_rtl_to_presentation(presentation)
+ # CJK font hint so the target script renders with a proper
+ # typeface instead of shape-dependent fallbacks.
+ _apply_ea_font_hints(presentation, target_language)
+
if translate_images:
try:
self._translate_images(presentation, target_language)
@@ -405,12 +481,30 @@ class PowerPointTranslator:
non_empty = [t for t in texts if t and t.strip()]
self._translation_stats["attempted"] += len(non_empty)
+ from services.translation_tm import translate_with_tm
+
+ provider_name = (
+ self._provider.get_name() if hasattr(self._provider, "get_name")
+ else type(self._provider).__name__
+ ) if self._provider is not None else "legacy"
+
if self._provider is not None:
- translated = self._translate_with_provider(
- texts, target_language, source_language
- )
+ def _do_translate(miss_texts):
+ return self._translate_with_provider(
+ miss_texts, target_language, source_language
+ )
else:
- translated = self._translate_with_legacy(texts, target_language, source_language)
+ def _do_translate(miss_texts):
+ return self._translate_with_legacy(
+ miss_texts, target_language, source_language
+ )
+
+ # Translation memory: reuse this user's previous translations
+ # (identical context/prompt) before hitting the provider.
+ translated = translate_with_tm(
+ texts, target_language, source_language,
+ provider_name, getattr(self, "_tm_scope", None), _do_translate,
+ )
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
self._translation_stats["changed"] += changed
@@ -743,7 +837,14 @@ class PowerPointTranslator:
return None
def _apply_chart_translations(self, output_path: Path) -> None:
- """Re-inject chart text translations by modifying chart XML parts.
+ """Re-inject chart text translations into the saved .pptx ZIP.
+
+ python-pptx's ChartPart exposes a read-only ``blob`` property, so an
+ in-memory `chart_part.blob = ...` assignment fails silently and the
+ chart text never reaches the output file. Instead — exactly like the
+ Word translator — we translate into a fresh parse of each chart
+ part's XML and rewrite the corresponding ZIP entries of the
+ already-saved output file.
Matching strategy: prefer the stored `element_path` (set at collect
time) to navigate directly to the right element. Fall back to
@@ -759,6 +860,9 @@ class PowerPointTranslator:
total_translated = 0
total_skipped = 0
+ # partname (e.g. "ppt/charts/chart1.xml") → updated XML bytes
+ updated_parts: Dict[str, bytes] = {}
+
for chart_data in self._chart_entries:
entries = chart_data['entries']
chart_part = chart_data['chart_part']
@@ -804,17 +908,39 @@ class PowerPointTranslator:
target.text = leading + (entry['translated'] or '').strip() + trailing
total_translated += 1
- # Update the chart part blob
- chart_part.blob = etree.tostring(
- chart_xml,
- xml_declaration=True,
- encoding='UTF-8',
- standalone=True,
- )
+ part_name = str(getattr(chart_part, "partname", "")).lstrip("/")
+ if part_name:
+ updated_parts[part_name] = etree.tostring(
+ chart_xml,
+ xml_declaration=True,
+ encoding='UTF-8',
+ standalone=True,
+ )
except Exception as e:
_log_error("pptx_chart_update_error", error=str(e))
+ # Single ZIP rewrite pass for all updated chart parts
+ if updated_parts:
+ import zipfile
+ import io as _io
+ try:
+ with zipfile.ZipFile(output_path, 'r') as zf_in:
+ buf = _io.BytesIO()
+ with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf_out:
+ for item in zf_in.namelist():
+ data = updated_parts.get(item, zf_in.read(item))
+ zf_out.writestr(item, data)
+ with open(output_path, 'wb') as f:
+ f.write(buf.getvalue())
+ _log_info(
+ "pptx_charts_rewritten",
+ chart_parts=len(updated_parts),
+ translated=total_translated,
+ )
+ except Exception as e:
+ _log_error("pptx_chart_zip_rewrite_error", error=str(e))
+
# Clean up
self._chart_entries = []
diff --git a/translators/word_translator.py b/translators/word_translator.py
index 5ef09c0..f46e5d4 100644
--- a/translators/word_translator.py
+++ b/translators/word_translator.py
@@ -31,6 +31,73 @@ RTL_LANGUAGES: frozenset = frozenset(
{"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"}
)
+# East-Asian / complex-script font hints: when the target language uses
+# glyphs a Latin theme font lacks, Word falls back to a substitute —
+# setting the eastAsia (CJK) or cs (Arabic script) typeface keeps the
+# rendering consistent across runs.
+CJK_EASTASIA_FONTS: dict = {
+ "zh": "SimSun",
+ "zh-CN": "SimSun",
+ "zh-TW": "PMingLiU",
+ "ja": "Yu Mincho",
+ "ko": "Batang",
+}
+CS_FONTS: dict = {
+ "ar": "Arial",
+ "he": "Arial",
+ "fa": "Arial",
+ "ur": "Arial",
+}
+
+
+def _font_hints_for_target(target_language: str):
+ """(eastAsia_font, cs_font) hints for the target language, if any."""
+ code = (target_language or "").strip()
+ base = code.split("-")[0].lower()
+ return CJK_EASTASIA_FONTS.get(code) or CJK_EASTASIA_FONTS.get(base), CS_FONTS.get(base)
+
+
+def _apply_font_hints(document: Document, target_language: str) -> None:
+ """Set eastAsia/cs typeface hints on every run for CJK/Arabic targets.
+
+ Blanket application is safe: the hint only affects the glyphs of that
+ script, which Latin text does not contain.
+ """
+ eastasia, cs = _font_hints_for_target(target_language)
+ if not eastasia and not cs:
+ return
+
+ runs = []
+ for para in document.paragraphs:
+ runs.extend(para.runs)
+ for table in document.tables:
+ for row in table.rows:
+ for cell in row.cells:
+ for para in cell.paragraphs:
+ runs.extend(para.runs)
+ for section in document.sections:
+ for hf in (section.header, section.footer):
+ for para in hf.paragraphs:
+ runs.extend(para.runs)
+
+ hinted = 0
+ for run in runs:
+ rPr = run._r.get_or_add_rPr()
+ rFonts = rPr.find(qn("w:rFonts"))
+ if rFonts is None:
+ rFonts = OxmlElement("w:rFonts")
+ rPr.insert(0, rFonts)
+ if eastasia and not rFonts.get(qn("w:eastAsia")):
+ rFonts.set(qn("w:eastAsia"), eastasia)
+ hinted += 1
+ if cs and not rFonts.get(qn("w:cs")):
+ rFonts.set(qn("w:cs"), cs)
+ hinted += 1
+
+ if hinted:
+ from core.logging import get_logger as _gl
+ _gl(__name__).info("word_font_hints_applied", runs=hinted, eastasia=eastasia, cs=cs)
+
from core.logging import get_logger
@@ -62,7 +129,9 @@ def _set_paragraph_rtl(paragraph: Paragraph) -> None:
Sets:
- w:pPr/w:bidi → paragraph text direction = RTL
- - w:pPr/w:jc → alignment = right
+ - w:pPr/w:jc → mirrored alignment (left→right), ONLY when the
+ paragraph has no explicit alignment — centered/justified titles
+ must not be forced right-aligned.
- w:rPr/w:rtl → run-level RTL marker for each run
"""
pPr = paragraph._p.get_or_add_pPr()
@@ -71,10 +140,12 @@ def _set_paragraph_rtl(paragraph: Paragraph) -> None:
pPr.append(OxmlElement("w:bidi"))
jc = pPr.find(qn("w:jc"))
- if jc is None:
- jc = OxmlElement("w:jc")
- pPr.append(jc)
- jc.set(qn("w:val"), "right")
+ explicit_alignment = jc is not None and jc.get(qn("w:val")) not in (None, "", "left")
+ if not explicit_alignment:
+ if jc is None:
+ jc = OxmlElement("w:jc")
+ pPr.append(jc)
+ jc.set(qn("w:val"), "right")
for run in paragraph.runs:
rPr = run._r.get_or_add_rPr()
@@ -173,6 +244,7 @@ class WordTranslator:
self._provider = provider
self._custom_prompt: Optional[str] = None
self._translation_stats = {"attempted": 0, "changed": 0}
+ self._tm_scope = None # set via set_tm_scope (per-user translation memory)
def set_provider(self, provider: TranslationProvider) -> None:
"""Set the translation provider."""
@@ -182,6 +254,12 @@ class WordTranslator:
"""Set custom system prompt for LLM providers."""
self._custom_prompt = prompt
+ def set_tm_scope(self, user_id: Optional[str], prompt: Optional[str] = None) -> None:
+ """Enable the per-user translation memory for this job."""
+ from services.translation_tm import TMScope
+
+ self._tm_scope = TMScope.from_prompt(user_id, prompt or self._custom_prompt)
+
def translate_file(
self,
input_path: Path,
@@ -333,6 +411,10 @@ class WordTranslator:
if target_language.lower() in RTL_LANGUAGES:
_apply_rtl_to_document(document)
+ # CJK / Arabic-script font hints so Word renders the target
+ # script with a proper typeface instead of per-run fallbacks.
+ _apply_font_hints(document, target_language)
+
if progress_callback:
progress_callback(
{
@@ -461,12 +543,30 @@ class WordTranslator:
non_empty = [t for t in texts if t and t.strip()]
self._translation_stats["attempted"] += len(non_empty)
+ from services.translation_tm import translate_with_tm
+
+ provider_name = (
+ self._provider.get_name() if hasattr(self._provider, "get_name")
+ else type(self._provider).__name__
+ ) if self._provider is not None else "legacy"
+
if self._provider is not None:
- translated = self._translate_with_provider(
- texts, target_language, source_language
- )
+ def _do_translate(miss_texts):
+ return self._translate_with_provider(
+ miss_texts, target_language, source_language
+ )
else:
- translated = self._translate_with_legacy(texts, target_language, source_language)
+ def _do_translate(miss_texts):
+ return self._translate_with_legacy(
+ miss_texts, target_language, source_language
+ )
+
+ # Translation memory: reuse this user's previous translations
+ # (identical context/prompt) before hitting the provider.
+ translated = translate_with_tm(
+ texts, target_language, source_language,
+ provider_name, self._tm_scope, _do_translate,
+ )
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
self._translation_stats["changed"] += changed
@@ -541,12 +641,19 @@ class WordTranslator:
Handles: paragraphs, tables, SDT (TOC/index), text boxes, shapes,
AlternateContent blocks, and any nested drawing elements.
+
+ A single ``seen_run_elements`` set is shared by every collector so
+ runs living in text boxes are never collected twice (the paragraph
+ walk descends into w:txbxContent too).
"""
count_before = len(text_elements)
+ seen_run_elements: set = set()
# Pass 1: walk direct body children
for element in document.element.body:
- self._collect_from_element(element, document, text_elements)
+ self._collect_from_element(
+ element, document, text_elements, seen_run_elements
+ )
pass1_count = len(text_elements) - count_before
@@ -554,15 +661,18 @@ class WordTranslator:
# Text boxes / rectangles / shapes store their text here, nested deep
# inside → → → or
# inside → → .
- self._collect_from_textboxes(document.element.body, document, text_elements)
+ self._collect_from_textboxes(
+ document.element.body, document, text_elements, seen_run_elements
+ )
pass2_count = len(text_elements) - count_before - pass1_count
- # Pass 3: footnotes and endnotes (live in separate parts)
+ # Pass 3: footnotes, endnotes and comments (live in separate parts)
if post_save_callbacks is None:
post_save_callbacks = []
self._collect_from_footnotes(document, text_elements, post_save_callbacks)
self._collect_from_endnotes(document, text_elements, post_save_callbacks)
+ self._collect_from_comments(document, text_elements, post_save_callbacks)
total = len(text_elements) - count_before
_log_info(
@@ -573,34 +683,38 @@ class WordTranslator:
)
def _collect_from_element(
- self, element, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
+ self, element, document: Document,
+ text_elements: List[Tuple[str, Callable[[str], None]]],
+ seen_run_elements: Optional[set] = None,
) -> None:
"""Recursively collect from any element type."""
if isinstance(element, CT_P):
paragraph = Paragraph(element, document)
- self._collect_from_paragraph(paragraph, text_elements)
+ self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
elif isinstance(element, CT_Tbl):
table = Table(element, document)
- self._collect_from_table(table, text_elements)
+ self._collect_from_table(table, text_elements, seen_run_elements)
elif element.tag == qn("w:sdt"):
- self._collect_from_sdt(element, document, text_elements)
+ self._collect_from_sdt(element, document, text_elements, seen_run_elements)
elif element.tag == self._TAG_ALT_CONTENT:
# wraps drawing/shape content
for part in element:
- self._collect_from_element(part, document, text_elements)
+ self._collect_from_element(part, document, text_elements, seen_run_elements)
else:
# For any other container element, recurse into children
# to catch paragraphs nested in unexpected wrappers
for child in element:
if isinstance(child, CT_P):
paragraph = Paragraph(child, document)
- self._collect_from_paragraph(paragraph, text_elements)
+ self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
elif isinstance(child, CT_Tbl):
table = Table(child, document)
- self._collect_from_table(table, text_elements)
+ self._collect_from_table(table, text_elements, seen_run_elements)
def _collect_from_textboxes(
- self, root, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
+ self, root, document: Document,
+ text_elements: List[Tuple[str, Callable[[str], None]]],
+ seen_run_elements: Optional[set] = None,
) -> None:
"""Find and collect text from ALL elements in the XML tree.
@@ -612,20 +726,23 @@ class WordTranslator:
- Shapes nested in blocks
The element contains regular paragraphs
- with runs, just like normal body text.
+ with runs, just like normal body text. Runs already collected
+ during the body walk are skipped via ``seen_run_elements``.
"""
# Find all w:txbxContent elements anywhere in the tree
for txbx in root.iter(qn("w:txbxContent")):
for child in txbx:
if isinstance(child, CT_P):
paragraph = Paragraph(child, document)
- self._collect_from_paragraph(paragraph, text_elements)
+ self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
elif isinstance(child, CT_Tbl):
table = Table(child, document)
- self._collect_from_table(table, text_elements)
+ self._collect_from_table(table, text_elements, seen_run_elements)
def _collect_from_sdt(
- self, sdt_element, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
+ self, sdt_element, document: Document,
+ text_elements: List[Tuple[str, Callable[[str], None]]],
+ seen_run_elements: Optional[set] = None,
) -> None:
"""Collect text from Structured Document Tags (TOC, index, content controls).
@@ -645,10 +762,10 @@ class WordTranslator:
for child in sdt_content:
if isinstance(child, CT_P):
paragraph = Paragraph(child, document)
- self._collect_from_paragraph(paragraph, text_elements)
+ self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
elif isinstance(child, CT_Tbl):
table = Table(child, document)
- self._collect_from_table(table, text_elements)
+ self._collect_from_table(table, text_elements, seen_run_elements)
def _collect_from_footnotes(
self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]],
@@ -783,6 +900,59 @@ class WordTranslator:
post_save_callbacks.append(write_endnotes_back)
+ def _collect_from_comments(
+ self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]],
+ post_save_callbacks: List[Callable[[Path], None]] = None,
+ ) -> None:
+ """Collect text from comments/balloons (word/comments.xml part).
+
+ Same mechanism as footnotes: the comments part is separate from the
+ main document tree, so translations are written back after save.
+ """
+ comments_xml = self._find_part_by_content_type(
+ document,
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml",
+ )
+ if comments_xml is None:
+ return
+
+ collected = 0
+ for t_elem in comments_xml.iter(qn("w:t")):
+ original = t_elem.text or ""
+ if not original.strip():
+ continue
+
+ def make_t_setter(t):
+ def setter(text: str) -> None:
+ t.text = text
+ return setter
+
+ text_elements.append((original, make_t_setter(t_elem)))
+ collected += 1
+
+ if collected and post_save_callbacks is not None:
+ def write_comments_back(output_path: Path) -> None:
+ try:
+ new_blob = etree.tostring(
+ comments_xml,
+ xml_declaration=True,
+ encoding="UTF-8",
+ standalone=True,
+ )
+ tmp_path = output_path.with_suffix(".tmp_com")
+ with zipfile.ZipFile(output_path, "r") as zin, \
+ zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zout:
+ for item in zin.namelist():
+ if item == "word/comments.xml":
+ zout.writestr(item, new_blob)
+ else:
+ zout.writestr(item, zin.read(item))
+ tmp_path.replace(output_path)
+ except Exception as e:
+ _log_error("word_comments_writeback_error", error=str(e))
+
+ post_save_callbacks.append(write_comments_back)
+
def _collect_from_charts(
self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
) -> None:
@@ -1144,23 +1314,39 @@ class WordTranslator:
except Exception as e:
_log_error("word_diagram_zip_rewrite_error", error=str(e))
+ @staticmethod
+ def _rpr_signature(run_element) -> str:
+ """Formatting signature of a run: serialized rPr XML (or "")."""
+ rpr = run_element.find(qn("w:rPr"))
+ if rpr is None:
+ return ""
+ import lxml.etree as _et
+
+ return _et.tostring(rpr, encoding="unicode")
+
def _collect_from_paragraph(
self,
paragraph: Paragraph,
text_elements: List[Tuple[str, Callable[[str], None]]],
+ seen_run_elements: Optional[set] = None,
) -> None:
"""Collect text from paragraph runs, preserving inter-run whitespace.
- Each run is sent for translation WITHOUT its surrounding whitespace.
- The whitespace is captured and reapplied after translation so that words
- at formatting boundaries (e.g. bold/normal) do not get concatenated.
+ Adjacent runs sharing the SAME parent element and the SAME run
+ formatting (rPr) are merged into ONE translation unit: the sentence
+ is translated whole — not fragment by fragment — and the result is
+ written into the first run while the sibling runs are blanked.
+ This is what keeps mid-sentence bold spans ("This is *very*
+ important") coherent in the target language, like DeepL's inline
+ tag handling.
Note: python-docx's `paragraph.runs` only returns DIRECT child
elements, not those inside (used for TOC entries,
cross-references, bookmark links). We therefore iterate the full
- XML tree to find every and use a set of element ids to
- deduplicate — this avoids translating the same run twice while
- ensuring hyperlink text IS picked up.
+ XML tree to find every and deduplicate by element identity —
+ `seen_run_elements` is shared across paragraphs so runs living in
+ text boxes (also collected by _collect_from_textboxes) are not
+ translated twice.
"""
# Check full paragraph text including nested content (hyperlinks, etc.)
full_text = ''.join(
@@ -1169,33 +1355,83 @@ class WordTranslator:
if not full_text:
return
- # Collect every element in the paragraph tree, including
- # those nested in , , etc. The dedup by
- # element id is defensive — `paragraph.runs` and the manual iter
- # below could overlap if python-docx starts surfacing nested runs.
- seen_run_ids: set = set()
+ if seen_run_elements is None:
+ seen_run_elements = set()
- # 1) Direct runs (paragraph.runs is the python-docx-native API).
- for run in paragraph.runs:
- run_id = id(run._r)
- if run_id in seen_run_ids:
- continue
- seen_run_ids.add(run_id)
- if run.text and run.text.strip():
- self._append_run_translation(run, text_elements)
-
- # 2) Runs nested inside (TOC, cross-references).
- # python-docx's `paragraph.runs` does NOT descend into hyperlinks in
- # version 1.x — we have to walk the XML ourselves.
+ # Every in the paragraph tree, in document order, deduplicated
+ # by element identity (paragraph.runs and the manual iter overlap).
+ ordered_runs = []
for r_elem in paragraph._p.iter(qn('w:r')):
- run_id = id(r_elem)
- if run_id in seen_run_ids:
+ if id(r_elem) in seen_run_elements:
continue
- seen_run_ids.add(run_id)
- # Build a Run wrapper so the setter API is consistent.
- run = Run(r_elem, paragraph)
- if run.text and run.text.strip():
+ seen_run_elements.add(id(r_elem))
+ ordered_runs.append(r_elem)
+
+ # Merge adjacent runs: same parent + same formatting signature.
+ # Merging never crosses a parent boundary, so runs belonging to
+ # different hyperlinks stay separate units.
+ group: list = [] # list of r_elems
+ group_signature: Optional[str] = None
+
+ def _flush_group():
+ combined = "".join(
+ (t.text or "")
+ for r in group
+ for t in r.findall(qn("w:t"))
+ )
+ if not combined.strip():
+ return
+ non_empty = [r for r in group if r.findall(qn("w:t"))]
+ if len(non_empty) == 1:
+ run = Run(non_empty[0], paragraph)
self._append_run_translation(run, text_elements)
+ return
+ leading = combined[: len(combined) - len(combined.lstrip())]
+ trailing = combined[len(combined.rstrip()):]
+ stripped = combined.strip()
+ if not stripped:
+ return
+
+ first = non_empty[0]
+
+ def make_group_setter(first_r, siblings, lead: str, trail: str):
+ def setter(text: str) -> None:
+ from docx.text.run import Run as _Run
+
+ run = _Run(first_r, paragraph)
+ # Reapply the group's boundary whitespace so words are
+ # never concatenated with the next differently-formatted
+ # run ("This is quite" + "very" → "quite very").
+ run.text = lead + text.strip() + trail
+ # Blank the merged siblings: the whole sentence now
+ # lives in the first run (formatting is identical).
+ for sib in siblings:
+ for t_elem in sib.findall(qn("w:t")):
+ t_elem.text = ""
+
+ return setter
+
+ siblings = non_empty[1:]
+ text_elements.append(
+ (stripped, make_group_setter(first, siblings, leading, trailing))
+ )
+
+ for r_elem in ordered_runs:
+ # Whitespace-only runs join the group: dropping them would
+ # concatenate words ("Hello" + " " + "World" → "HelloWorld").
+ # They carry no w:t text, so a group of only whitespace runs is
+ # skipped at flush time by the strip() check.
+ signature = self._rpr_signature(r_elem)
+ same_parent = (
+ group and group[-1].getparent() is r_elem.getparent()
+ )
+ if group and same_parent and signature == group_signature:
+ group.append(r_elem)
+ else:
+ _flush_group()
+ group = [r_elem]
+ group_signature = signature
+ _flush_group()
def _append_run_translation(
self,
@@ -1220,15 +1456,16 @@ class WordTranslator:
text_elements.append((stripped, make_setter(run, leading, trailing)))
def _collect_from_table(
- self, table: Table, text_elements: List[Tuple[str, Callable[[str], None]]]
+ self, table: Table, text_elements: List[Tuple[str, Callable[[str], None]]],
+ seen_run_elements: Optional[set] = None,
) -> None:
"""Collect text from table cells."""
for row in table.rows:
for cell in row.cells:
for paragraph in cell.paragraphs:
- self._collect_from_paragraph(paragraph, text_elements)
+ self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
for nested_table in cell.tables:
- self._collect_from_table(nested_table, text_elements)
+ self._collect_from_table(nested_table, text_elements, seen_run_elements)
def _collect_from_section(
self, section: Section, text_elements: List[Tuple[str, Callable[[str], None]]]
diff --git a/utils/file_handler.py b/utils/file_handler.py
index a3e9ce8..6f53202 100644
--- a/utils/file_handler.py
+++ b/utils/file_handler.py
@@ -179,3 +179,41 @@ class FileHandler:
# Global file handler instance
file_handler = FileHandler()
+
+
+def validate_zip_safety(
+ file_path: Path,
+ max_compression_ratio: float = 100.0,
+ max_total_uncompressed_mb: int = 1024,
+) -> None:
+ """Reject ZIP-based documents that expand dangerously (zip bombs).
+
+ Office files (.xlsx/.docx/.pptx) are ZIP archives: a small uploaded file
+ can decompress to gigabytes in memory. Raises ValueError when the file
+ is not a readable archive, expands beyond the allowed ratio, or when the
+ total uncompressed size exceeds the cap.
+ """
+ import zipfile
+
+ max_total_bytes = max_total_uncompressed_mb * 1024 * 1024
+ try:
+ with zipfile.ZipFile(file_path) as zf:
+ total_uncompressed = 0
+ for info in zf.infolist():
+ if info.is_dir():
+ continue
+ total_uncompressed += info.file_size
+ if (
+ info.compress_size > 0
+ and info.file_size / info.compress_size > max_compression_ratio
+ ):
+ raise ValueError(
+ f"Entry '{info.filename}' expands more than "
+ f"{int(max_compression_ratio)}x its compressed size."
+ )
+ if total_uncompressed > max_total_bytes:
+ raise ValueError(
+ f"Archive expands beyond {max_total_uncompressed_mb} MB."
+ )
+ except zipfile.BadZipFile as e:
+ raise ValueError(f"Not a valid ZIP-based document: {e}")