feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s

Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions

View File

@@ -14,6 +14,7 @@ from fastapi.responses import FileResponse, JSONResponse
from config import config
from utils import file_handler
from utils.file_handler import validate_zip_safety
from middleware.api_key_auth import get_authenticated_user
logger = logging.getLogger(__name__)
@@ -32,7 +33,9 @@ def _resolve_model(
@router.get("/providers/available")
async def get_available_providers():
async def get_available_providers(
current_user: Optional[Any] = Depends(get_authenticated_user),
):
"""
Return every provider that is enabled — checking BOTH the admin settings JSON
AND environment variables (env vars act as a fallback / override).
@@ -42,8 +45,12 @@ async def get_available_providers():
- Ollama is only shown in DEV mode (APP_ENV=development or SHOW_OLLAMA=true).
- openrouter → shown as "Traduction IA Essentielle" (cheap models).
- openrouter_premium → shown as "Traduction IA Premium" (premium models).
- Filtered to the engines included in the caller's plan
(PLANS[plan]["providers"]) so the UI never offers an engine the
translate endpoint would reject.
"""
from routes.admin_routes import load_settings
from models.subscription import PlanType, PLANS
settings = load_settings()
is_dev = os.getenv("APP_ENV", "production").lower() == "development"
@@ -189,6 +196,16 @@ async def get_available_providers():
# Filter to the engines included in the caller's plan (anonymous → Free).
user_plan = PlanType.FREE
if current_user is not None:
try:
user_plan = PlanType(getattr(current_user, "plan", PlanType.FREE))
except ValueError:
user_plan = PlanType.FREE
allowed = set((PLANS.get(user_plan) or PLANS[PlanType.FREE]).get("providers", []))
available = [p for p in available if p.get("id") in allowed]
return JSONResponse(
status_code=200,
headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
@@ -198,49 +215,36 @@ async def get_available_providers():
@router.get("/languages")
async def get_supported_languages():
"""Get list of supported language codes, ordered by internet popularity"""
"""Get list of supported language codes, ordered by internet popularity.
Served from LanguageValidator (single source of truth): the 35 most
requested languages first, then the rest of the validated ISO 639-1 set
alphabetically. "auto" is excluded — it is a source-only value handled
by the source_lang parameter.
"""
from middleware.validation import LanguageValidator
popular_order = [
# Top 5 — dominant on the internet
"en", "es", "de", "fr", "ja",
# Top 6-15
"pt", "ru", "it", "zh-CN", "zh-TW", "pl", "nl", "tr", "ko", "ar",
# Top 16-25
"fa", "vi", "id", "uk", "sv", "cs", "el", "he", "hi", "ro",
# Next most requested
"da", "fi", "no", "hu", "th", "sk", "bg", "hr", "ca", "ms", "zh",
]
names = LanguageValidator.LANGUAGE_NAMES
supported = [
c for c in LanguageValidator.SUPPORTED_LANGUAGES if c != "auto"
]
ordered = [c for c in popular_order if c in supported]
ordered += sorted(c for c in supported if c not in ordered)
return {
"supported_languages": {
# Top 5 — dominant on the internet
"en": "English",
"es": "Spanish",
"de": "German",
"fr": "French",
"ja": "Japanese",
# Top 6-15
"pt": "Portuguese",
"ru": "Russian",
"it": "Italian",
"zh-CN": "Chinese (Simplified)",
"zh-TW": "Chinese (Traditional)",
"pl": "Polish",
"nl": "Dutch",
"tr": "Turkish",
"ko": "Korean",
"ar": "Arabic",
# Top 16-25
"fa": "Persian (Farsi)",
"vi": "Vietnamese",
"id": "Indonesian",
"uk": "Ukrainian",
"sv": "Swedish",
"cs": "Czech",
"el": "Greek",
"he": "Hebrew",
"hi": "Hindi",
"ro": "Romanian",
# Others
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"hu": "Hungarian",
"th": "Thai",
"sk": "Slovak",
"bg": "Bulgarian",
"hr": "Croatian",
"ca": "Catalan",
"ms": "Malay",
},
"supported_languages": {code: names.get(code, code.upper()) for code in ordered},
"count": len(ordered),
"note": "Supported languages may vary depending on the translation service configured",
}
@@ -276,6 +280,10 @@ async def translate_batch_documents(
file_handler.save_upload_file(file, input_path)
# Zip bomb protection: Office files are ZIP archives
if file_extension != ".pdf":
validate_zip_safety(input_path)
if file_extension == ".xlsx":
excel_translator.translate_file(
input_path, output_path, target_language, source_language
@@ -300,6 +308,18 @@ async def translate_batch_documents(
}
)
except ValueError as e:
file_handler.cleanup_file(input_path)
logger.warning(f"Rejected unsafe or invalid archive: {file.filename}: {e}")
results.append(
{
"filename": file.filename,
"status": "error",
"error": "CORRUPTED_FILE",
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
}
)
except Exception as e:
logger.exception(f"Error processing {file.filename}")
results.append(
@@ -341,6 +361,9 @@ async def extract_texts_from_document(
input_path = config.UPLOAD_DIR / input_filename
file_handler.save_upload_file(file, input_path)
if file_extension != ".pdf":
validate_zip_safety(input_path)
texts = []
if file_extension == ".xlsx":
@@ -423,6 +446,17 @@ async def extract_texts_from_document(
except HTTPException:
raise
except ValueError as e:
file_handler.cleanup_file(input_path)
logger.warning(f"Text extraction rejected unsafe or invalid archive: {file.filename}: {e}")
return JSONResponse(
status_code=400,
content={
"error": "CORRUPTED_FILE",
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
},
)
except Exception as e:
logger.exception("Text extraction error")
return JSONResponse(