feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
This commit is contained in:
@@ -14,6 +14,7 @@ from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from config import config
|
||||
from utils import file_handler
|
||||
from utils.file_handler import validate_zip_safety
|
||||
from middleware.api_key_auth import get_authenticated_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,7 +33,9 @@ def _resolve_model(
|
||||
|
||||
|
||||
@router.get("/providers/available")
|
||||
async def get_available_providers():
|
||||
async def get_available_providers(
|
||||
current_user: Optional[Any] = Depends(get_authenticated_user),
|
||||
):
|
||||
"""
|
||||
Return every provider that is enabled — checking BOTH the admin settings JSON
|
||||
AND environment variables (env vars act as a fallback / override).
|
||||
@@ -42,8 +45,12 @@ async def get_available_providers():
|
||||
- Ollama is only shown in DEV mode (APP_ENV=development or SHOW_OLLAMA=true).
|
||||
- openrouter → shown as "Traduction IA Essentielle" (cheap models).
|
||||
- openrouter_premium → shown as "Traduction IA Premium" (premium models).
|
||||
- Filtered to the engines included in the caller's plan
|
||||
(PLANS[plan]["providers"]) so the UI never offers an engine the
|
||||
translate endpoint would reject.
|
||||
"""
|
||||
from routes.admin_routes import load_settings
|
||||
from models.subscription import PlanType, PLANS
|
||||
|
||||
settings = load_settings()
|
||||
is_dev = os.getenv("APP_ENV", "production").lower() == "development"
|
||||
@@ -189,6 +196,16 @@ async def get_available_providers():
|
||||
|
||||
|
||||
|
||||
# Filter to the engines included in the caller's plan (anonymous → Free).
|
||||
user_plan = PlanType.FREE
|
||||
if current_user is not None:
|
||||
try:
|
||||
user_plan = PlanType(getattr(current_user, "plan", PlanType.FREE))
|
||||
except ValueError:
|
||||
user_plan = PlanType.FREE
|
||||
allowed = set((PLANS.get(user_plan) or PLANS[PlanType.FREE]).get("providers", []))
|
||||
available = [p for p in available if p.get("id") in allowed]
|
||||
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
|
||||
@@ -198,49 +215,36 @@ async def get_available_providers():
|
||||
|
||||
@router.get("/languages")
|
||||
async def get_supported_languages():
|
||||
"""Get list of supported language codes, ordered by internet popularity"""
|
||||
"""Get list of supported language codes, ordered by internet popularity.
|
||||
|
||||
Served from LanguageValidator (single source of truth): the 35 most
|
||||
requested languages first, then the rest of the validated ISO 639-1 set
|
||||
alphabetically. "auto" is excluded — it is a source-only value handled
|
||||
by the source_lang parameter.
|
||||
"""
|
||||
from middleware.validation import LanguageValidator
|
||||
|
||||
popular_order = [
|
||||
# Top 5 — dominant on the internet
|
||||
"en", "es", "de", "fr", "ja",
|
||||
# Top 6-15
|
||||
"pt", "ru", "it", "zh-CN", "zh-TW", "pl", "nl", "tr", "ko", "ar",
|
||||
# Top 16-25
|
||||
"fa", "vi", "id", "uk", "sv", "cs", "el", "he", "hi", "ro",
|
||||
# Next most requested
|
||||
"da", "fi", "no", "hu", "th", "sk", "bg", "hr", "ca", "ms", "zh",
|
||||
]
|
||||
|
||||
names = LanguageValidator.LANGUAGE_NAMES
|
||||
supported = [
|
||||
c for c in LanguageValidator.SUPPORTED_LANGUAGES if c != "auto"
|
||||
]
|
||||
ordered = [c for c in popular_order if c in supported]
|
||||
ordered += sorted(c for c in supported if c not in ordered)
|
||||
|
||||
return {
|
||||
"supported_languages": {
|
||||
# Top 5 — dominant on the internet
|
||||
"en": "English",
|
||||
"es": "Spanish",
|
||||
"de": "German",
|
||||
"fr": "French",
|
||||
"ja": "Japanese",
|
||||
# Top 6-15
|
||||
"pt": "Portuguese",
|
||||
"ru": "Russian",
|
||||
"it": "Italian",
|
||||
"zh-CN": "Chinese (Simplified)",
|
||||
"zh-TW": "Chinese (Traditional)",
|
||||
"pl": "Polish",
|
||||
"nl": "Dutch",
|
||||
"tr": "Turkish",
|
||||
"ko": "Korean",
|
||||
"ar": "Arabic",
|
||||
# Top 16-25
|
||||
"fa": "Persian (Farsi)",
|
||||
"vi": "Vietnamese",
|
||||
"id": "Indonesian",
|
||||
"uk": "Ukrainian",
|
||||
"sv": "Swedish",
|
||||
"cs": "Czech",
|
||||
"el": "Greek",
|
||||
"he": "Hebrew",
|
||||
"hi": "Hindi",
|
||||
"ro": "Romanian",
|
||||
# Others
|
||||
"da": "Danish",
|
||||
"fi": "Finnish",
|
||||
"no": "Norwegian",
|
||||
"hu": "Hungarian",
|
||||
"th": "Thai",
|
||||
"sk": "Slovak",
|
||||
"bg": "Bulgarian",
|
||||
"hr": "Croatian",
|
||||
"ca": "Catalan",
|
||||
"ms": "Malay",
|
||||
},
|
||||
"supported_languages": {code: names.get(code, code.upper()) for code in ordered},
|
||||
"count": len(ordered),
|
||||
"note": "Supported languages may vary depending on the translation service configured",
|
||||
}
|
||||
|
||||
@@ -276,6 +280,10 @@ async def translate_batch_documents(
|
||||
|
||||
file_handler.save_upload_file(file, input_path)
|
||||
|
||||
# Zip bomb protection: Office files are ZIP archives
|
||||
if file_extension != ".pdf":
|
||||
validate_zip_safety(input_path)
|
||||
|
||||
if file_extension == ".xlsx":
|
||||
excel_translator.translate_file(
|
||||
input_path, output_path, target_language, source_language
|
||||
@@ -300,6 +308,18 @@ async def translate_batch_documents(
|
||||
}
|
||||
)
|
||||
|
||||
except ValueError as e:
|
||||
file_handler.cleanup_file(input_path)
|
||||
logger.warning(f"Rejected unsafe or invalid archive: {file.filename}: {e}")
|
||||
results.append(
|
||||
{
|
||||
"filename": file.filename,
|
||||
"status": "error",
|
||||
"error": "CORRUPTED_FILE",
|
||||
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
|
||||
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing {file.filename}")
|
||||
results.append(
|
||||
@@ -341,6 +361,9 @@ async def extract_texts_from_document(
|
||||
input_path = config.UPLOAD_DIR / input_filename
|
||||
file_handler.save_upload_file(file, input_path)
|
||||
|
||||
if file_extension != ".pdf":
|
||||
validate_zip_safety(input_path)
|
||||
|
||||
texts = []
|
||||
|
||||
if file_extension == ".xlsx":
|
||||
@@ -423,6 +446,17 @@ async def extract_texts_from_document(
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except ValueError as e:
|
||||
file_handler.cleanup_file(input_path)
|
||||
logger.warning(f"Text extraction rejected unsafe or invalid archive: {file.filename}: {e}")
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "CORRUPTED_FILE",
|
||||
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
|
||||
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Text extraction error")
|
||||
return JSONResponse(
|
||||
|
||||
Reference in New Issue
Block a user