feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s

Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions

View File

@@ -285,6 +285,67 @@ async def get_admin_dashboard(admin_id: str = Depends(require_admin)):
"last_check": None,
}
# OCR status (scanned PDFs): configured via admin settings or env key.
try:
from services.mistral_ocr import MistralOCRClient
settings = load_settings()
mistral_key = (settings.mistral.api_key or "").strip() or os.getenv(
"MISTRAL_API_KEY", ""
).strip()
ocr_configured = bool(mistral_key) and (
settings.mistral.enabled
# enabled=False in a saved-but-untouched settings file must not
# hide an env-configured OCR (same rule as the translate route)
or not (settings.mistral.api_key or "").strip()
)
providers_status["mistral_ocr"] = {
"name": "mistral_ocr",
"available": ocr_configured,
"error": None
if ocr_configured
else "Non configuré — les PDF scannés seront refusés (MISTRAL_API_KEY)",
"last_check": None,
# No live call here: health is derived from configuration.
"config_only": True,
}
except Exception as e:
providers_status["mistral_ocr"] = {
"name": "mistral_ocr",
"available": False,
"error": str(e)[:100],
"last_check": None,
}
# LLM/translation engines configured via admin settings (key presence
# only — no live call, keys are never exposed).
try:
settings = load_settings()
def _engine_status(section_name: str, env_var: str, label: str):
section = getattr(settings, section_name, None)
key_set = bool(
((getattr(section, "api_key", None) or "").strip())
or os.getenv(env_var, "").strip()
)
providers_status[section_name] = {
"name": section_name,
"available": key_set,
"error": None if key_set else f"Clé absente ({env_var})",
"last_check": None,
"config_only": True,
"label": label,
}
_engine_status("deepl", "DEEPL_API_KEY", "DeepL")
_engine_status("openrouter", "OPENROUTER_API_KEY", "Traduction IA Éco")
_engine_status("openrouter_premium", "OPENROUTER_API_KEY", "Traduction IA Premium")
_engine_status("openai", "OPENAI_API_KEY", "OpenAI")
_engine_status("zai", "ZAI_API_KEY", "Grok (xAI)")
_engine_status("google_cloud", "GOOGLE_CLOUD_API_KEY", "Google Cloud")
except Exception as e:
logger.warning(f"admin dashboard engines status failed: {e}")
return {
"timestamp": health_status.get("timestamp"),
"status": health_status.get("status"),
@@ -641,13 +702,24 @@ async def update_default_provider(
provider: str = Form(...),
admin_id: str = Depends(require_admin),
):
"""Update the default translation provider"""
"""Update the default translation provider.
The allowed list mirrors the providers actually wired in the translate
route (and declared in SettingsConfig) so the admin can set any of them as
the default — previously google_cloud, openrouter_premium, deepseek and
minimax were wrongly rejected here even though they are fully supported.
"""
valid_providers = [
"google",
"google_cloud",
"deepl",
"openai",
"openrouter",
"openrouter_premium",
"deepseek",
"minimax",
"zai",
# Mode aliases resolved at translation time.
"classic",
"llm",
]
@@ -852,6 +924,7 @@ class SettingsConfig(BaseModel):
deepseek: ProviderSettings = ProviderSettings()
minimax: ProviderSettings = ProviderSettings()
zai: ProviderSettings = ProviderSettings()
mistral: ProviderSettings = ProviderSettings() # OCR Mistral (PDF scannés)
smtp: SmtpSettings = SmtpSettings()
fallback_chain: str = "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
fallback_chain_classic: str = "google,google_cloud,deepl"
@@ -924,6 +997,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
payload["zai"] = _merge_env(settings.zai, key_env="ZAI_API_KEY", model_env="ZAI_MODEL", url_env="ZAI_BASE_URL", default_model="grok-2-1212", default_url="https://api.x.ai/v1")
payload["google_cloud"] = _merge_env(settings.google_cloud, key_env="GOOGLE_CLOUD_API_KEY")
payload["mistral"] = _merge_env(settings.mistral, key_env="MISTRAL_API_KEY", model_env="MISTRAL_OCR_MODEL", default_model="mistral-ocr-latest")
# SMTP: merge from env vars, but never expose password
smtp_data = settings.smtp.model_dump()
@@ -958,6 +1032,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
"zai": bool(os.getenv("ZAI_API_KEY", "").strip()),
"google_cloud": bool(os.getenv("GOOGLE_CLOUD_API_KEY", "").strip()),
"mistral": bool(os.getenv("MISTRAL_API_KEY", "").strip()),
"smtp": bool(os.getenv("SMTP_HOST", "").strip()),
}
return JSONResponse(
@@ -1029,6 +1104,49 @@ async def test_provider(
status_code=200, content={"available": True, "test_result": result}
)
elif provider == "mistral":
api_key = _key(provider_config.api_key, "MISTRAL_API_KEY")
if not api_key:
return JSONResponse(
status_code=400,
content={
"available": False,
"error": "Aucune clé API Mistral trouvée (JSON ou .env MISTRAL_API_KEY).",
},
)
import requests as _requests
resp = _requests.get(
"https://api.mistral.ai/v1/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10,
)
if resp.ok:
n_models = len(resp.json().get("data", []))
return JSONResponse(
status_code=200,
content={
"available": True,
"test_result": f"Clé valide — {n_models} modèles accessibles",
},
)
elif resp.status_code in (401, 403):
return JSONResponse(
status_code=resp.status_code,
content={
"available": False,
"error": f"Clé API Mistral invalide (HTTP {resp.status_code}).",
},
)
else:
return JSONResponse(
status_code=500,
content={
"available": False,
"error": f"Erreur Mistral HTTP {resp.status_code}: {resp.text[:200]}",
},
)
elif provider == "google_cloud":
api_key = _key(provider_config.api_key, "GOOGLE_CLOUD_API_KEY")
if not api_key:

View File

@@ -15,6 +15,7 @@ from routes.admin_routes import router as admin_router
from routes.legacy_routes import router as legacy_router
from routes.glossary_routes import router as glossary_router
from routes.prompt_routes import router as prompt_router
from routes.waitlist_routes import router as waitlist_router
router.include_router(translate_router, tags=["Translation"])
router.include_router(auth_router, tags=["Authentication"])
@@ -23,3 +24,4 @@ router.include_router(admin_router, tags=["Admin"])
router.include_router(legacy_router, tags=["Legacy"])
router.include_router(glossary_router, tags=["Glossaries"])
router.include_router(prompt_router, tags=["Prompts"])
router.include_router(waitlist_router, tags=["Waitlist"])

View File

@@ -14,6 +14,7 @@ from fastapi.responses import FileResponse, JSONResponse
from config import config
from utils import file_handler
from utils.file_handler import validate_zip_safety
from middleware.api_key_auth import get_authenticated_user
logger = logging.getLogger(__name__)
@@ -32,7 +33,9 @@ def _resolve_model(
@router.get("/providers/available")
async def get_available_providers():
async def get_available_providers(
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
Return every provider that is enabled — checking BOTH the admin settings JSON
AND environment variables (env vars act as a fallback / override).
@@ -42,8 +45,12 @@ async def get_available_providers():
- Ollama is only shown in DEV mode (APP_ENV=development or SHOW_OLLAMA=true).
- openrouter → shown as "Traduction IA Essentielle" (cheap models).
- openrouter_premium → shown as "Traduction IA Premium" (premium models).
- Filtered to the engines included in the caller's plan
(PLANS[plan]["providers"]) so the UI never offers an engine the
translate endpoint would reject.
"""
from routes.admin_routes import load_settings
from models.subscription import PlanType, PLANS
settings = load_settings()
is_dev = os.getenv("APP_ENV", "production").lower() == "development"
@@ -189,6 +196,16 @@ async def get_available_providers():
# Filter to the engines included in the caller's plan (anonymous → Free).
user_plan = PlanType.FREE
if current_user is not None:
try:
user_plan = PlanType(getattr(current_user, "plan", PlanType.FREE))
except ValueError:
user_plan = PlanType.FREE
allowed = set((PLANS.get(user_plan) or PLANS[PlanType.FREE]).get("providers", []))
available = [p for p in available if p.get("id") in allowed]
return JSONResponse(
status_code=200,
headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
@@ -198,49 +215,36 @@ async def get_available_providers():
@router.get("/languages")
async def get_supported_languages():
"""Get list of supported language codes, ordered by internet popularity"""
"""Get list of supported language codes, ordered by internet popularity.
Served from LanguageValidator (single source of truth): the 35 most
requested languages first, then the rest of the validated ISO 639-1 set
alphabetically. "auto" is excluded — it is a source-only value handled
by the source_lang parameter.
"""
from middleware.validation import LanguageValidator
popular_order = [
# Top 5 — dominant on the internet
"en", "es", "de", "fr", "ja",
# Top 6-15
"pt", "ru", "it", "zh-CN", "zh-TW", "pl", "nl", "tr", "ko", "ar",
# Top 16-25
"fa", "vi", "id", "uk", "sv", "cs", "el", "he", "hi", "ro",
# Next most requested
"da", "fi", "no", "hu", "th", "sk", "bg", "hr", "ca", "ms", "zh",
]
names = LanguageValidator.LANGUAGE_NAMES
supported = [
c for c in LanguageValidator.SUPPORTED_LANGUAGES if c != "auto"
]
ordered = [c for c in popular_order if c in supported]
ordered += sorted(c for c in supported if c not in ordered)
return {
"supported_languages": {
# Top 5 — dominant on the internet
"en": "English",
"es": "Spanish",
"de": "German",
"fr": "French",
"ja": "Japanese",
# Top 6-15
"pt": "Portuguese",
"ru": "Russian",
"it": "Italian",
"zh-CN": "Chinese (Simplified)",
"zh-TW": "Chinese (Traditional)",
"pl": "Polish",
"nl": "Dutch",
"tr": "Turkish",
"ko": "Korean",
"ar": "Arabic",
# Top 16-25
"fa": "Persian (Farsi)",
"vi": "Vietnamese",
"id": "Indonesian",
"uk": "Ukrainian",
"sv": "Swedish",
"cs": "Czech",
"el": "Greek",
"he": "Hebrew",
"hi": "Hindi",
"ro": "Romanian",
# Others
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"hu": "Hungarian",
"th": "Thai",
"sk": "Slovak",
"bg": "Bulgarian",
"hr": "Croatian",
"ca": "Catalan",
"ms": "Malay",
},
"supported_languages": {code: names.get(code, code.upper()) for code in ordered},
"count": len(ordered),
"note": "Supported languages may vary depending on the translation service configured",
}
@@ -276,6 +280,10 @@ async def translate_batch_documents(
file_handler.save_upload_file(file, input_path)
# Zip bomb protection: Office files are ZIP archives
if file_extension != ".pdf":
validate_zip_safety(input_path)
if file_extension == ".xlsx":
excel_translator.translate_file(
input_path, output_path, target_language, source_language
@@ -300,6 +308,18 @@ async def translate_batch_documents(
}
)
except ValueError as e:
file_handler.cleanup_file(input_path)
logger.warning(f"Rejected unsafe or invalid archive: {file.filename}: {e}")
results.append(
{
"filename": file.filename,
"status": "error",
"error": "CORRUPTED_FILE",
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
}
)
except Exception as e:
logger.exception(f"Error processing {file.filename}")
results.append(
@@ -341,6 +361,9 @@ async def extract_texts_from_document(
input_path = config.UPLOAD_DIR / input_filename
file_handler.save_upload_file(file, input_path)
if file_extension != ".pdf":
validate_zip_safety(input_path)
texts = []
if file_extension == ".xlsx":
@@ -423,6 +446,17 @@ async def extract_texts_from_document(
except HTTPException:
raise
except ValueError as e:
file_handler.cleanup_file(input_path)
logger.warning(f"Text extraction rejected unsafe or invalid archive: {file.filename}: {e}")
return JSONResponse(
status_code=400,
content={
"error": "CORRUPTED_FILE",
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
},
)
except Exception as e:
logger.exception("Text extraction error")
return JSONResponse(

View File

@@ -9,6 +9,7 @@ Story 3.6: Documentation OpenAPI complète avec exemples et codes d'erreur
import os
import re
import secrets
import uuid
import time
import socket
@@ -43,7 +44,7 @@ from typing_extensions import Annotated
from config import config
from translators import ExcelTranslator, WordTranslator, PowerPointTranslator
from models.subscription import PlanType
from models.subscription import PlanType, PLANS
from services.auth_service import (
record_usage,
check_usage_limits,
@@ -54,6 +55,7 @@ from middleware.tier_quota import _seconds_until_next_month, _next_month_utc
from middleware.validation import FileValidator, ValidationError, LanguageValidator, webhook_validator
from middleware.api_key_auth import get_authenticated_user, get_user_from_api_key
from utils import file_handler
from utils.file_handler import validate_zip_safety
# Import models from schemas (Story 3.6 - DRY principle)
from schemas.translation import (
@@ -148,6 +150,43 @@ def _tier_for_quota(plan) -> str:
return "free"
def _tier_from_plan_str(plan_str: Optional[str]) -> str:
"""Map the job's ``user_plan`` string to a quota tier.
``user_plan`` is built with ``str(current_user.plan)`` which renders a
str-Enum as "PlanType.PRO" — accept both that repr and the raw value.
"""
name = (plan_str or "").strip().lower()
if name.startswith("plantype."):
name = name.split(".", 1)[1]
try:
return _tier_for_quota(PlanType(name))
except ValueError:
return "free"
def _plan_from_user(current_user: Optional[Any]) -> PlanType:
"""Resolve the request's plan (anonymous requests are on the Free plan)."""
if current_user is None:
return PlanType.FREE
plan = getattr(current_user, "plan", PlanType.FREE)
try:
return PlanType(plan)
except ValueError:
return PlanType.FREE
def _allowed_providers_for_plan(plan: PlanType) -> set:
"""Engines included in the plan (source of truth: PLANS[plan]["providers"])."""
plan_cfg = PLANS.get(plan) or PLANS[PlanType.FREE]
return set(plan_cfg.get("providers", []))
def _image_translation_allowed_for_plan(plan: PlanType) -> bool:
"""Vision (text-in-images) is sold on Pro and above."""
return plan in (PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE)
def _next_midnight_utc() -> datetime:
"""Get next midnight UTC."""
now = datetime.now(timezone.utc)
@@ -215,6 +254,21 @@ def _parse_content_disposition(content_disp: str) -> Optional[str]:
return None
def _sanitize_url_filename(filename: str) -> str:
"""Strip path traversal / control chars from a remote-provided filename."""
filename = Path(filename).name
filename = re.sub(r"[\x00-\x1f\x7f-\x9f]", "", filename)
filename = re.sub(r'[<>:"/\\|?*]', "_", filename)
# Drop leading dots (hidden files, ".." leftovers)
filename = filename.lstrip(".")
if len(filename) > 255:
name, ext = (
filename.rsplit(".", 1) if "." in filename else (filename, "")
)
filename = name[:250] + ("." + ext if ext else "")
return filename or "downloaded_file"
def _is_ssrf_risk(hostname: str) -> bool:
"""Return True if hostname resolves to a private/reserved IP (SSRF prevention).
@@ -252,95 +306,130 @@ async def download_from_url(url: str, timeout: int = 30) -> tuple[Path, str]:
details={"scheme": parsed_url.scheme or "none"},
)
hostname = parsed_url.hostname or ""
if not hostname or _is_ssrf_risk(hostname):
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message="The URL points to a blocked address (private or internal network).",
details={"reason": "ssrf_blocked"},
)
def _validate_url_target(u: str):
p = urlparse(u)
if p.scheme not in ("http", "https"):
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message="Only HTTP/HTTPS URLs are accepted.",
details={"scheme": p.scheme or "none"},
)
h = p.hostname or ""
if not h or _is_ssrf_risk(h):
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message="The URL points to a blocked address (private or internal network).",
details={"reason": "ssrf_blocked"},
)
_validate_url_target(url)
MAX_REDIRECTS = 5
try:
async with httpx.AsyncClient(
timeout=timeout, follow_redirects=True, max_redirects=10
timeout=timeout, follow_redirects=False
) as client:
async with client.stream("GET", url) as response:
if response.status_code != 200:
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message=f"URL unreachable (HTTP {response.status_code})",
details={"status_code": response.status_code, "url": url[:100]},
)
content_length = response.headers.get("content-length")
if content_length:
try:
file_size = int(content_length)
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
if file_size > max_size_bytes:
current_url = url
for _ in range(MAX_REDIRECTS + 1):
_validate_url_target(current_url)
async with client.stream("GET", current_url) as response:
if response.status_code in (301, 302, 303, 307, 308):
location = response.headers.get("location", "")
if not location:
raise TranslateEndpointError(
code=TranslateEndpointError.FILE_TOO_LARGE,
message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
details={
"size_mb": round(file_size / (1024 * 1024), 2),
"max_mb": MAX_FILE_SIZE_MB,
},
code=TranslateEndpointError.URL_UNREACHABLE,
message="Redirect without Location header.",
details={"reason": "invalid_redirect"},
)
except ValueError:
pass
# Re-check the redirect target before following it
current_url = str(httpx.URL(current_url).join(location))
continue
filename = None
content_disp = response.headers.get("content-disposition", "")
if content_disp:
filename = _parse_content_disposition(content_disp)
if response.status_code != 200:
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message=f"URL unreachable (HTTP {response.status_code})",
details={"status_code": response.status_code, "url": url[:100]},
)
if not filename:
filename = unquote(Path(parsed_url.path).name) or "downloaded_file"
content_length = response.headers.get("content-length")
if content_length:
try:
file_size = int(content_length)
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
if file_size > max_size_bytes:
raise TranslateEndpointError(
code=TranslateEndpointError.FILE_TOO_LARGE,
message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
details={
"size_mb": round(file_size / (1024 * 1024), 2),
"max_mb": MAX_FILE_SIZE_MB,
},
)
except ValueError:
pass
extension = Path(filename).suffix.lower()
if extension not in ACCEPTED_EXTENSIONS:
raise TranslateEndpointError(
code=TranslateEndpointError.INVALID_FORMAT,
details={
"detected_extension": extension or "none",
"accepted_formats": list(ACCEPTED_EXTENSIONS),
},
)
filename = None
content_disp = response.headers.get("content-disposition", "")
if content_disp:
filename = _parse_content_disposition(content_disp)
unique_id = str(uuid.uuid4())[:8]
safe_filename = f"{unique_id}_{filename}"
temp_path = config.UPLOAD_DIR / safe_filename
if not filename:
filename = unquote(Path(urlparse(current_url).path).name)
temp_path.parent.mkdir(parents=True, exist_ok=True)
filename = _sanitize_url_filename(filename)
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
downloaded_bytes = 0
extension = Path(filename).suffix.lower()
if extension not in ACCEPTED_EXTENSIONS:
raise TranslateEndpointError(
code=TranslateEndpointError.INVALID_FORMAT,
details={
"detected_extension": extension or "none",
"accepted_formats": list(ACCEPTED_EXTENSIONS),
},
)
async with aiofiles.open(temp_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=65536):
downloaded_bytes += len(chunk)
unique_id = str(uuid.uuid4())[:8]
safe_filename = f"{unique_id}_{filename}"
temp_path = config.UPLOAD_DIR / safe_filename
if downloaded_bytes > max_size_bytes:
await f.close()
if temp_path.exists():
temp_path.unlink()
raise TranslateEndpointError(
code=TranslateEndpointError.FILE_TOO_LARGE,
details={
"size_mb": round(
downloaded_bytes / (1024 * 1024), 2
),
"max_mb": MAX_FILE_SIZE_MB,
},
)
temp_path.parent.mkdir(parents=True, exist_ok=True)
await f.write(chunk)
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
downloaded_bytes = 0
async with aiofiles.open(temp_path, "rb") as f:
header = await f.read(4)
await validate_file_content(header, extension)
async with aiofiles.open(temp_path, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=65536):
downloaded_bytes += len(chunk)
return temp_path, filename
if downloaded_bytes > max_size_bytes:
await f.close()
if temp_path.exists():
temp_path.unlink()
raise TranslateEndpointError(
code=TranslateEndpointError.FILE_TOO_LARGE,
details={
"size_mb": round(
downloaded_bytes / (1024 * 1024), 2
),
"max_mb": MAX_FILE_SIZE_MB,
},
)
await f.write(chunk)
async with aiofiles.open(temp_path, "rb") as f:
header = await f.read(4)
await validate_file_content(header, extension)
return temp_path, filename
raise TranslateEndpointError(
code=TranslateEndpointError.URL_UNREACHABLE,
message="Too many redirects.",
details={"reason": "too_many_redirects", "max_redirects": MAX_REDIRECTS},
)
except httpx.TimeoutException:
if temp_path and temp_path.exists():
@@ -420,9 +509,13 @@ def _cleanup_old_jobs() -> None:
return
_last_cleanup_ts = current_time
# Snapshot the items before filtering: concurrent coroutines (background
# translation workers, status pollers) mutate _translation_jobs, and
# iterating a dict while it is resized raises
# "RuntimeError: dictionary changed size during iteration".
expired_job_ids = [
job_id
for job_id, job in _translation_jobs.items()
for job_id, job in list(_translation_jobs.items())
if job.get("status") in ("completed", "failed")
and (
(ts := job.get("completed_at") or job.get("failed_at"))
@@ -444,6 +537,68 @@ def _job_age_seconds(timestamp_str: str) -> float:
return 0.0
def _provider_model(provider: Any) -> str:
"""Best-effort read of a provider's model name.
The new-style providers (``services/providers/*``) store the model in
``self._model``, while the legacy providers (``services/translation_service``)
expose ``self.model``. Read either so cost-factor logic works for both.
"""
if provider is None:
return ""
return (
getattr(provider, "_model", None)
or getattr(provider, "model", None)
or ""
)
def _compute_cost_factor(provider: Any, provider_name: str = "") -> int:
"""Billing cost factor (1 = standard, 5 = premium).
Premium models (Claude, GPT-4 family, etc.) cost more and are billed at a
higher factor. Cheap variants are explicitly downgraded to 1 (``haiku``,
and the small GPT-4 models such as ``gpt-4o-mini`` / ``gpt-4o-nano``).
The provider may be passed by instance (model read off it) or only by
name (e.g. the ``openrouter_premium`` alias).
"""
model_lower = _provider_model(provider).lower()
provider_lower = (provider_name or "").lower()
if "haiku" in model_lower or "mini" in model_lower or "nano" in model_lower:
return 1
if any(k in model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
return 5
return 1
def _compute_duration_seconds(created_at_iso: str) -> float:
"""Elapsed seconds since ``created_at`` (UTC ISO string).
Uses a timezone-aware timestamp() to avoid the local-time bug that
``time.mktime`` introduced. Falls back to 0 on parse errors so a bad
timestamp can never flip a successful job into the error branch.
"""
return _job_age_seconds(created_at_iso)
async def _release_quota_if_needed(user_id, usage_recorded: bool, job_id: str) -> None:
"""Release the reserved translation quota on a soft-failure path.
The translation worker reserves a quota slot at request time. The generic
``except`` branch releases it on hard failures, but the early ``return``
paths (empty output / no translatable text / 0 texts translated) are NOT
exceptions and must release the quota explicitly — otherwise the user
loses a slot without receiving a translation.
"""
if user_id and not usage_recorded:
try:
await asyncio.to_thread(release_translation_quota, user_id)
logger.info(f"Job {job_id}: released reserved quota after soft-failure")
except Exception as release_err:
logger.exception(f"Job {job_id}: failed to release reserved quota: {release_err}")
@router_v1.post(
"/translate",
response_model=TranslateResponse,
@@ -476,6 +631,14 @@ async def translate_document_v1(
pdf_mode: Optional[Literal["layout", "text_only"]] = Form(
default=None, description="PDF translation mode: 'layout' (preserve layout) or 'text_only' (clean text output). PDF only."
),
formality: Optional[Literal["formal", "informal"]] = Form(
default=None,
description="Tone override for LLM engines: 'formal' or 'informal'. Ignored by classic engines.",
),
output_mode: Optional[Literal["single", "bilingual"]] = Form(
default="single",
description="Output mode: 'single' (translated file only) or 'bilingual' (docx with source + translation interleaved).",
),
translate_images: bool = Form(
default=False, description="Translate text inside images using AI vision"
),
@@ -676,7 +839,9 @@ async def translate_document_v1(
rate_limit_remaining = -1
try:
LanguageValidator.validate(target_lang)
# Keep the canonical form (e.g. "zh-CN") so providers receive a
# code they understand instead of the raw user input.
target_lang = LanguageValidator.validate(target_lang)
except ValidationError as e:
raise TranslateEndpointError(
code="INVALID_FORMAT",
@@ -686,7 +851,7 @@ async def translate_document_v1(
if source_lang and source_lang != "auto":
try:
LanguageValidator.validate(source_lang)
source_lang = LanguageValidator.validate(source_lang)
except ValidationError:
raise TranslateEndpointError(
code="INVALID_FORMAT",
@@ -760,7 +925,23 @@ async def translate_document_v1(
details={"error": "sha256_calculation_failed"},
)
# Office files are ZIP archives: reject archives that expand far
# beyond their uploaded size (zip bomb protection).
if file_extension != ".pdf":
try:
validate_zip_safety(input_path)
except ValueError as e:
file_handler_util.cleanup_file(input_path)
raise TranslateEndpointError(
code=TranslateEndpointError.CORRUPTED_FILE,
message="The file is invalid or expands dangerously.",
details={"reason": "unsafe_archive", "detail": str(e)[:200]},
)
job_id = f"tr_{uuid.uuid4().hex[:12]}"
# Secret per-job token: required to follow/download a job that has no
# logged-in owner (anonymous API calls).
job_access_token = secrets.token_urlsafe(24)
# Track file metadata in Redis with TTL
await storage_tracker.track_file(
@@ -771,6 +952,7 @@ async def translate_document_v1(
"file_hash": file_hash,
"input_path": str(input_path),
"user_id": str(user_id) if user_id else None,
"access_token": job_access_token,
"timestamp": datetime.now(timezone.utc).isoformat(),
},
)
@@ -794,6 +976,7 @@ async def translate_document_v1(
"target_lang": target_lang,
"created_at": datetime.now(timezone.utc).isoformat(),
"user_id": user_id,
"access_token": job_access_token,
"input_path": str(input_path),
"file_extension": file_extension,
"provider": provider or mode,
@@ -803,6 +986,8 @@ async def translate_document_v1(
"prompt_id": prompt_id, # Story 3.12: Store prompt_id
"pdf_mode": pdf_mode, # PDF translation mode
"translate_images": translate_images,
"formality": formality, # LLM tone override
"output_mode": output_mode or "single", # single | bilingual
}
await set_job_status_async(job_id, _translation_jobs[job_id])
@@ -821,6 +1006,39 @@ async def translate_document_v1(
)
provider_to_use = "google"
# ── Plan-based engine gating (grille commerciale : PLANS[plan]["providers"]) ──
# Chaque plan n'expose que ses moteurs ; un moteur hors plan est refusé
# (403) au lieu d'être exécuté aux frais de la maison.
_user_plan = _plan_from_user(current_user)
if provider_to_use not in _allowed_providers_for_plan(_user_plan):
raise TranslateEndpointError(
code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
message=(
f"Le moteur « {provider_to_use} » n'est pas inclus dans votre plan. "
"Passez à un plan supérieur pour l'utiliser."
),
details={
"feature": "provider",
"provider": provider_to_use,
"plan": str(_user_plan.value),
"allowed_providers": sorted(_allowed_providers_for_plan(_user_plan)),
},
)
# La traduction du texte dans les images (vision) est vendue Pro et plus.
if translate_images and not _image_translation_allowed_for_plan(_user_plan):
raise TranslateEndpointError(
code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
message=(
"La traduction du texte dans les images est réservée "
"aux plans Pro et supérieurs."
),
details={
"feature": "translate_images",
"plan": str(_user_plan.value),
},
)
asyncio.create_task(
_run_translation_job(
job_id=job_id,
@@ -837,6 +1055,8 @@ async def translate_document_v1(
user_plan=str(current_user.plan) if current_user else "free",
pdf_mode=pdf_mode,
translate_images=translate_images,
formality=formality,
output_mode=output_mode or "single",
)
)
@@ -853,6 +1073,8 @@ async def translate_document_v1(
"file_name": original_filename,
"source_lang": source_lang,
"target_lang": target_lang,
# Needed to poll/download this job when calling without login
"access_token": job_access_token,
},
"meta": {
"rate_limit_remaining": rate_limit_remaining,
@@ -980,6 +1202,8 @@ async def _run_translation_job(
user_plan: Optional[str] = None, # Plan name for watermark decision
pdf_mode: Optional[str] = None, # PDF translation mode: "layout" or "text_only"
translate_images: bool = False,
formality: Optional[str] = None, # LLM tone override: formal/informal
output_mode: str = "single", # single | bilingual
) -> None:
"""
Run translation job in background with progress tracking.
@@ -1076,11 +1300,13 @@ async def _run_translation_job(
# Use custom_prompt if no prompt_id
effective_prompt = custom_prompt
# Build the full prompt combining effective prompt and glossary
# Build the full prompt combining effective prompt, glossary,
# formality directive and regional variant hint.
full_prompt = build_full_prompt(
effective_prompt, glossary_terms,
source_lang=glossary_source_lang, target_lang=target_lang,
glossary_target_lang=glossary_target_lang,
formality=formality,
)
from services.providers.google_provider import GoogleTranslationProvider
@@ -1259,6 +1485,8 @@ async def _run_translation_job(
job_translator = ExcelTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
if hasattr(job_translator, "set_tm_scope"):
job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1272,6 +1500,8 @@ async def _run_translation_job(
job_translator = WordTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
if hasattr(job_translator, "set_tm_scope"):
job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1285,6 +1515,8 @@ async def _run_translation_job(
job_translator = PowerPointTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
if hasattr(job_translator, "set_tm_scope"):
job_translator.set_tm_scope(user_id, full_prompt)
await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1299,6 +1531,40 @@ async def _run_translation_job(
job_translator = PDFTranslator(provider=translation_provider)
if hasattr(job_translator, "set_custom_prompt"):
job_translator.set_custom_prompt(full_prompt)
if hasattr(job_translator, "set_tm_scope"):
job_translator.set_tm_scope(user_id, full_prompt)
# OCR (PDF scannés) : réglages admin > variables d'env.
mistral_cfg = getattr(_admin_cfg, "mistral", None)
mistral_key = _cfg(
getattr(mistral_cfg, "api_key", None), "MISTRAL_API_KEY"
)
# Only honor the admin "enabled" flag when Mistral is actually
# configured there — a freshly saved settings file carries
# enabled=false defaults that must not override env-based OCR.
mistral_admin_configured = bool(
mistral_cfg is not None
and (getattr(mistral_cfg, "api_key", None) or "").strip()
)
job_translator.set_ocr_config(
api_key=mistral_key,
model=_cfg(
getattr(mistral_cfg, "model", None),
"MISTRAL_OCR_MODEL",
"mistral-ocr-latest",
),
timeout=int(
_cfg(
str(getattr(mistral_cfg, "timeout", "") or ""),
"MISTRAL_OCR_TIMEOUT",
"180",
)
),
enabled=(
getattr(mistral_cfg, "enabled", None)
if mistral_admin_configured
else None
),
)
actual_output = await asyncio.to_thread(
job_translator.translate_file,
input_path,
@@ -1320,19 +1586,21 @@ async def _run_translation_job(
error_msg = "Translation failed: output file is empty or missing. The translation provider may be unavailable."
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
stats = job_translator.get_translation_stats()
attempted = stats.get("attempted", 0)
changed = stats.get("changed", 0)
if attempted == 0 and file_extension in ('.docx', '.xlsx', '.pptx'):
if attempted == 0:
error_msg = (
"Aucun texte traduisible détecté dans le document. "
"Le fichier est peut-être vide, protégé, ou ne contient que des images."
)
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
if attempted > 0:
@@ -1346,6 +1614,7 @@ async def _run_translation_job(
)
logger.error(f"Job {job_id}: {error_msg}")
tracker.set_error(error_msg)
await _release_quota_if_needed(user_id, usage_recorded, job_id)
return
elif ratio < 0.05:
# Very suspicious — likely partial failure, warn but don't block
@@ -1430,9 +1699,7 @@ async def _run_translation_job(
from middleware.metrics import record_translation_retry
record_translation_retry(
reason="l1_fail",
tier=_tier_for_quota(
current_user.plan if current_user else None
) if current_user else "free",
tier=_tier_from_plan_str(user_plan),
)
except Exception:
pass
@@ -1448,9 +1715,7 @@ async def _run_translation_job(
from middleware.metrics import record_translation_retry
record_translation_retry(
reason="l0_fail",
tier=_tier_for_quota(
current_user.plan if current_user else None
) if current_user else "free",
tier=_tier_from_plan_str(user_plan),
)
except Exception:
pass
@@ -1467,9 +1732,7 @@ async def _run_translation_job(
from services.quality import run_l2_check
# Tier gate: Pro+ plans only (unless gate is disabled)
user_tier = (
_tier_for_quota(current_user.plan) if current_user else "free"
)
user_tier = _tier_from_plan_str(user_plan)
tier_gate_on = getattr(config, "QUALITY_L2_TIER_GATE", True)
if not tier_gate_on or user_tier in ("pro", "business", "enterprise"):
translated_chunks_for_l2 = [s["translated"] for s in quality_samples]
@@ -1498,21 +1761,52 @@ async def _run_translation_job(
f"Job {job_id}: quality L2 layer failed: {l2_err}"
)
if user_id:
# Determine cost factor based on selected provider and model
cost_factor = 1
provider_lower = (provider or "").lower()
prov_model = ""
if translation_provider:
prov_model = getattr(translation_provider, "model", "") or ""
prov_model_lower = prov_model.lower()
if any(k in prov_model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
if "haiku" in prov_model_lower:
cost_factor = 1
# ------------------------------------------------------------------
# QA report (pure heuristics — numbers fidelity, untranslated
# ratio, 0-100 score). Never blocks the job; surfaced in the job
# status for the UI/API.
# ------------------------------------------------------------------
try:
from services.quality.qa_report import run_qa_report
qa = await asyncio.to_thread(
run_qa_report, input_path, output_path, target_lang, file_extension
)
if qa:
job["quality"] = qa
except Exception as qa_err:
logger.warning(f"Job {job_id}: QA report failed: {qa_err}")
# ------------------------------------------------------------------
# Bilingual output (docx): interleaves source paragraphs above
# their translation. Falls back silently to the translated file.
# ------------------------------------------------------------------
if output_mode == "bilingual" and file_extension == ".docx":
try:
from translators.bilingual import make_bilingual_docx
bilingual_path = output_path.with_name(
output_path.stem + "_bilingual" + output_path.suffix
)
actual = await asyncio.to_thread(
make_bilingual_docx, input_path, output_path, bilingual_path
)
if actual:
output_path = Path(actual)
logger.info(f"Job {job_id}: bilingual output generated")
else:
cost_factor = 5
logger.warning(
f"Job {job_id}: bilingual output skipped "
"(structure mismatch) — returning translated file"
)
except Exception as bi_err:
logger.warning(f"Job {job_id}: bilingual output failed: {bi_err}")
if user_id:
# Determine cost factor based on selected provider and model.
# _compute_cost_factor reads the model off the provider robustly
# (new-style providers store it in ``_model``, legacy in ``model``).
cost_factor = _compute_cost_factor(translation_provider, provider or "")
# Persist monthly usage counters in PostgreSQL (docs + pages)
pages = await asyncio.to_thread(
@@ -1537,7 +1831,7 @@ async def _run_translation_job(
tracker.set_completed(str(output_path))
# Record translation metric
duration = time.time() - time.mktime(datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).timetuple())
duration = _compute_duration_seconds(job.get("created_at", ""))
record_translation(provider=provider, file_type=file_extension or "unknown", duration=duration, status="success")
logger.info(f"Job {job_id}: Completed successfully")
@@ -1621,6 +1915,50 @@ async def _run_translation_job(
)
def _check_job_access(
job: dict, current_user, token: Optional[str]
) -> Optional[JSONResponse]:
"""Return an error response if the caller may not see this job, else None.
Jobs owned by a logged-in user require that same user. Anonymous jobs
require the secret per-job token returned when the job was created.
"""
job_user_id = job.get("user_id")
if job_user_id:
if not current_user:
return JSONResponse(
status_code=401,
content={
"error": "AUTH_REQUIRED",
"message": "Authentication is required to access this job.",
"details": {"job_id": job.get("id")},
},
)
if str(job_user_id) != str(current_user.id):
return JSONResponse(
status_code=403,
content={
"error": "ACCESS_DENIED",
"message": "You do not have access to this job.",
"details": {"job_id": job.get("id")},
},
)
return None
expected = job.get("access_token")
provided = token or ""
if not expected or not secrets.compare_digest(provided, expected):
return JSONResponse(
status_code=403,
content={
"error": "ACCESS_DENIED",
"message": "You do not have access to this job.",
"details": {"job_id": job.get("id"), "hint": "token_required"},
},
)
return None
@router_v1.get(
"/translations/{job_id}",
response_model=TranslationStatusResponse,
@@ -1631,6 +1969,7 @@ async def _run_translation_job(
)
async def get_translation_status(
job_id: str,
token: Optional[str] = None,
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
@@ -1680,6 +2019,10 @@ async def get_translation_status(
},
)
denied = _check_job_access(job, current_user, token)
if denied:
return denied
response_data = {
"id": job["id"],
"status": job["status"],
@@ -1689,6 +2032,7 @@ async def get_translation_status(
"source_lang": job.get("source_lang"),
"target_lang": job.get("target_lang"),
"created_at": job.get("created_at"),
"quality": job.get("quality"),
}
estimated_remaining = None
@@ -1768,6 +2112,7 @@ def _cleanup_files(input_path: Optional[str], output_path: Optional[str]) -> Non
)
async def download_translated_file(
job_id: str,
token: Optional[str] = None,
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
@@ -1819,16 +2164,9 @@ async def download_translated_file(
},
)
job_user_id = job.get("user_id")
if current_user and job_user_id and str(job_user_id) != str(current_user.id):
return JSONResponse(
status_code=403,
content={
"error": "ACCESS_DENIED",
"message": "You do not have access to this file.",
"details": {"job_id": job_id},
},
)
denied = _check_job_access(job, current_user, token)
if denied:
return denied
if job.get("status") != "completed":
return JSONResponse(

98
routes/waitlist_routes.py Normal file
View File

@@ -0,0 +1,98 @@
"""
Waitlist / email-capture routes for the marketing landing page.
Public endpoint: POST /api/v1/waitlist (no auth) — deduplicated by email,
persisted in data/waitlist.json (same JSON-file pattern as provider settings).
"""
import json
import logging
import threading
from datetime import datetime, timezone
from typing import List, Optional
from fastapi import APIRouter
from fastapi.responses import JSONResponse
from pydantic import BaseModel, EmailStr
from config import config
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/waitlist", tags=["Waitlist"])
WAITLIST_FILE = config.BASE_DIR / "data" / "waitlist.json"
_write_lock = threading.Lock()
class WaitlistEntry(BaseModel):
email: EmailStr
interest: Optional[str] = None
def _load_waitlist() -> List[dict]:
if not WAITLIST_FILE.exists():
return []
try:
with open(WAITLIST_FILE, encoding="utf-8") as f:
data = json.load(f)
return data if isinstance(data, list) else []
except Exception as e:
logger.warning(f"Failed to load waitlist: {e}")
return []
def _save_waitlist(entries: List[dict]) -> None:
WAITLIST_FILE.parent.mkdir(parents=True, exist_ok=True)
with open(WAITLIST_FILE, "w", encoding="utf-8") as f:
json.dump(entries, f, indent=2)
@router.post("", status_code=201, summary="Join the waitlist")
async def join_waitlist(entry: WaitlistEntry):
email = entry.email.lower().strip()
now = datetime.now(timezone.utc).isoformat()
with _write_lock:
entries = _load_waitlist()
for existing in entries:
if existing.get("email", "").lower() == email:
existing["updated_at"] = now
if entry.interest:
existing["interest"] = entry.interest
_save_waitlist(entries)
logger.info(f"Waitlist rejoin: {email}")
return JSONResponse(
status_code=200,
content={
"data": {"email": email, "status": "already_joined"},
"message": "You are already on the list.",
},
)
entries.append(
{
"email": email,
"interest": entry.interest,
"joined_at": now,
"updated_at": now,
}
)
_save_waitlist(entries)
logger.info(f"Waitlist join: {email} (total={len(entries)})")
return JSONResponse(
status_code=201,
content={
"data": {"email": email, "status": "joined"},
"message": "Welcome aboard!",
},
)
@router.get("/count", summary="Waitlist count")
async def waitlist_count():
return JSONResponse(
status_code=200,
content={"data": {"count": len(_load_waitlist())}},
)