feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
This commit is contained in:
@@ -285,6 +285,67 @@ async def get_admin_dashboard(admin_id: str = Depends(require_admin)):
|
||||
"last_check": None,
|
||||
}
|
||||
|
||||
# OCR status (scanned PDFs): configured via admin settings or env key.
|
||||
try:
|
||||
from services.mistral_ocr import MistralOCRClient
|
||||
|
||||
settings = load_settings()
|
||||
mistral_key = (settings.mistral.api_key or "").strip() or os.getenv(
|
||||
"MISTRAL_API_KEY", ""
|
||||
).strip()
|
||||
ocr_configured = bool(mistral_key) and (
|
||||
settings.mistral.enabled
|
||||
# enabled=False in a saved-but-untouched settings file must not
|
||||
# hide an env-configured OCR (same rule as the translate route)
|
||||
or not (settings.mistral.api_key or "").strip()
|
||||
)
|
||||
providers_status["mistral_ocr"] = {
|
||||
"name": "mistral_ocr",
|
||||
"available": ocr_configured,
|
||||
"error": None
|
||||
if ocr_configured
|
||||
else "Non configuré — les PDF scannés seront refusés (MISTRAL_API_KEY)",
|
||||
"last_check": None,
|
||||
# No live call here: health is derived from configuration.
|
||||
"config_only": True,
|
||||
}
|
||||
except Exception as e:
|
||||
providers_status["mistral_ocr"] = {
|
||||
"name": "mistral_ocr",
|
||||
"available": False,
|
||||
"error": str(e)[:100],
|
||||
"last_check": None,
|
||||
}
|
||||
|
||||
# LLM/translation engines configured via admin settings (key presence
|
||||
# only — no live call, keys are never exposed).
|
||||
try:
|
||||
settings = load_settings()
|
||||
|
||||
def _engine_status(section_name: str, env_var: str, label: str):
|
||||
section = getattr(settings, section_name, None)
|
||||
key_set = bool(
|
||||
((getattr(section, "api_key", None) or "").strip())
|
||||
or os.getenv(env_var, "").strip()
|
||||
)
|
||||
providers_status[section_name] = {
|
||||
"name": section_name,
|
||||
"available": key_set,
|
||||
"error": None if key_set else f"Clé absente ({env_var})",
|
||||
"last_check": None,
|
||||
"config_only": True,
|
||||
"label": label,
|
||||
}
|
||||
|
||||
_engine_status("deepl", "DEEPL_API_KEY", "DeepL")
|
||||
_engine_status("openrouter", "OPENROUTER_API_KEY", "Traduction IA Éco")
|
||||
_engine_status("openrouter_premium", "OPENROUTER_API_KEY", "Traduction IA Premium")
|
||||
_engine_status("openai", "OPENAI_API_KEY", "OpenAI")
|
||||
_engine_status("zai", "ZAI_API_KEY", "Grok (xAI)")
|
||||
_engine_status("google_cloud", "GOOGLE_CLOUD_API_KEY", "Google Cloud")
|
||||
except Exception as e:
|
||||
logger.warning(f"admin dashboard engines status failed: {e}")
|
||||
|
||||
return {
|
||||
"timestamp": health_status.get("timestamp"),
|
||||
"status": health_status.get("status"),
|
||||
@@ -641,13 +702,24 @@ async def update_default_provider(
|
||||
provider: str = Form(...),
|
||||
admin_id: str = Depends(require_admin),
|
||||
):
|
||||
"""Update the default translation provider"""
|
||||
"""Update the default translation provider.
|
||||
|
||||
The allowed list mirrors the providers actually wired in the translate
|
||||
route (and declared in SettingsConfig) so the admin can set any of them as
|
||||
the default — previously google_cloud, openrouter_premium, deepseek and
|
||||
minimax were wrongly rejected here even though they are fully supported.
|
||||
"""
|
||||
valid_providers = [
|
||||
"google",
|
||||
"google_cloud",
|
||||
"deepl",
|
||||
"openai",
|
||||
"openrouter",
|
||||
"openrouter_premium",
|
||||
"deepseek",
|
||||
"minimax",
|
||||
"zai",
|
||||
# Mode aliases resolved at translation time.
|
||||
"classic",
|
||||
"llm",
|
||||
]
|
||||
@@ -852,6 +924,7 @@ class SettingsConfig(BaseModel):
|
||||
deepseek: ProviderSettings = ProviderSettings()
|
||||
minimax: ProviderSettings = ProviderSettings()
|
||||
zai: ProviderSettings = ProviderSettings()
|
||||
mistral: ProviderSettings = ProviderSettings() # OCR Mistral (PDF scannés)
|
||||
smtp: SmtpSettings = SmtpSettings()
|
||||
fallback_chain: str = "google,google_cloud,deepl,openrouter,openrouter_premium,openai,deepseek,zai"
|
||||
fallback_chain_classic: str = "google,google_cloud,deepl"
|
||||
@@ -924,6 +997,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
|
||||
payload["zai"] = _merge_env(settings.zai, key_env="ZAI_API_KEY", model_env="ZAI_MODEL", url_env="ZAI_BASE_URL", default_model="grok-2-1212", default_url="https://api.x.ai/v1")
|
||||
|
||||
payload["google_cloud"] = _merge_env(settings.google_cloud, key_env="GOOGLE_CLOUD_API_KEY")
|
||||
payload["mistral"] = _merge_env(settings.mistral, key_env="MISTRAL_API_KEY", model_env="MISTRAL_OCR_MODEL", default_model="mistral-ocr-latest")
|
||||
|
||||
# SMTP: merge from env vars, but never expose password
|
||||
smtp_data = settings.smtp.model_dump()
|
||||
@@ -958,6 +1032,7 @@ async def get_settings(admin_id: str = Depends(require_admin)):
|
||||
"zai": bool(os.getenv("ZAI_API_KEY", "").strip()),
|
||||
|
||||
"google_cloud": bool(os.getenv("GOOGLE_CLOUD_API_KEY", "").strip()),
|
||||
"mistral": bool(os.getenv("MISTRAL_API_KEY", "").strip()),
|
||||
"smtp": bool(os.getenv("SMTP_HOST", "").strip()),
|
||||
}
|
||||
return JSONResponse(
|
||||
@@ -1029,6 +1104,49 @@ async def test_provider(
|
||||
status_code=200, content={"available": True, "test_result": result}
|
||||
)
|
||||
|
||||
elif provider == "mistral":
|
||||
api_key = _key(provider_config.api_key, "MISTRAL_API_KEY")
|
||||
if not api_key:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"available": False,
|
||||
"error": "Aucune clé API Mistral trouvée (JSON ou .env MISTRAL_API_KEY).",
|
||||
},
|
||||
)
|
||||
import requests as _requests
|
||||
|
||||
resp = _requests.get(
|
||||
"https://api.mistral.ai/v1/models",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok:
|
||||
n_models = len(resp.json().get("data", []))
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={
|
||||
"available": True,
|
||||
"test_result": f"Clé valide — {n_models} modèles accessibles",
|
||||
},
|
||||
)
|
||||
elif resp.status_code in (401, 403):
|
||||
return JSONResponse(
|
||||
status_code=resp.status_code,
|
||||
content={
|
||||
"available": False,
|
||||
"error": f"Clé API Mistral invalide (HTTP {resp.status_code}).",
|
||||
},
|
||||
)
|
||||
else:
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"available": False,
|
||||
"error": f"Erreur Mistral HTTP {resp.status_code}: {resp.text[:200]}",
|
||||
},
|
||||
)
|
||||
|
||||
elif provider == "google_cloud":
|
||||
api_key = _key(provider_config.api_key, "GOOGLE_CLOUD_API_KEY")
|
||||
if not api_key:
|
||||
|
||||
@@ -15,6 +15,7 @@ from routes.admin_routes import router as admin_router
|
||||
from routes.legacy_routes import router as legacy_router
|
||||
from routes.glossary_routes import router as glossary_router
|
||||
from routes.prompt_routes import router as prompt_router
|
||||
from routes.waitlist_routes import router as waitlist_router
|
||||
|
||||
router.include_router(translate_router, tags=["Translation"])
|
||||
router.include_router(auth_router, tags=["Authentication"])
|
||||
@@ -23,3 +24,4 @@ router.include_router(admin_router, tags=["Admin"])
|
||||
router.include_router(legacy_router, tags=["Legacy"])
|
||||
router.include_router(glossary_router, tags=["Glossaries"])
|
||||
router.include_router(prompt_router, tags=["Prompts"])
|
||||
router.include_router(waitlist_router, tags=["Waitlist"])
|
||||
|
||||
@@ -14,6 +14,7 @@ from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from config import config
|
||||
from utils import file_handler
|
||||
from utils.file_handler import validate_zip_safety
|
||||
from middleware.api_key_auth import get_authenticated_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,7 +33,9 @@ def _resolve_model(
|
||||
|
||||
|
||||
@router.get("/providers/available")
|
||||
async def get_available_providers():
|
||||
async def get_available_providers(
|
||||
current_user: Optional[Any] = Depends(get_authenticated_user),
|
||||
):
|
||||
"""
|
||||
Return every provider that is enabled — checking BOTH the admin settings JSON
|
||||
AND environment variables (env vars act as a fallback / override).
|
||||
@@ -42,8 +45,12 @@ async def get_available_providers():
|
||||
- Ollama is only shown in DEV mode (APP_ENV=development or SHOW_OLLAMA=true).
|
||||
- openrouter → shown as "Traduction IA Essentielle" (cheap models).
|
||||
- openrouter_premium → shown as "Traduction IA Premium" (premium models).
|
||||
- Filtered to the engines included in the caller's plan
|
||||
(PLANS[plan]["providers"]) so the UI never offers an engine the
|
||||
translate endpoint would reject.
|
||||
"""
|
||||
from routes.admin_routes import load_settings
|
||||
from models.subscription import PlanType, PLANS
|
||||
|
||||
settings = load_settings()
|
||||
is_dev = os.getenv("APP_ENV", "production").lower() == "development"
|
||||
@@ -189,6 +196,16 @@ async def get_available_providers():
|
||||
|
||||
|
||||
|
||||
# Filter to the engines included in the caller's plan (anonymous → Free).
|
||||
user_plan = PlanType.FREE
|
||||
if current_user is not None:
|
||||
try:
|
||||
user_plan = PlanType(getattr(current_user, "plan", PlanType.FREE))
|
||||
except ValueError:
|
||||
user_plan = PlanType.FREE
|
||||
allowed = set((PLANS.get(user_plan) or PLANS[PlanType.FREE]).get("providers", []))
|
||||
available = [p for p in available if p.get("id") in allowed]
|
||||
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
|
||||
@@ -198,49 +215,36 @@ async def get_available_providers():
|
||||
|
||||
@router.get("/languages")
|
||||
async def get_supported_languages():
|
||||
"""Get list of supported language codes, ordered by internet popularity"""
|
||||
"""Get list of supported language codes, ordered by internet popularity.
|
||||
|
||||
Served from LanguageValidator (single source of truth): the 35 most
|
||||
requested languages first, then the rest of the validated ISO 639-1 set
|
||||
alphabetically. "auto" is excluded — it is a source-only value handled
|
||||
by the source_lang parameter.
|
||||
"""
|
||||
from middleware.validation import LanguageValidator
|
||||
|
||||
popular_order = [
|
||||
# Top 5 — dominant on the internet
|
||||
"en", "es", "de", "fr", "ja",
|
||||
# Top 6-15
|
||||
"pt", "ru", "it", "zh-CN", "zh-TW", "pl", "nl", "tr", "ko", "ar",
|
||||
# Top 16-25
|
||||
"fa", "vi", "id", "uk", "sv", "cs", "el", "he", "hi", "ro",
|
||||
# Next most requested
|
||||
"da", "fi", "no", "hu", "th", "sk", "bg", "hr", "ca", "ms", "zh",
|
||||
]
|
||||
|
||||
names = LanguageValidator.LANGUAGE_NAMES
|
||||
supported = [
|
||||
c for c in LanguageValidator.SUPPORTED_LANGUAGES if c != "auto"
|
||||
]
|
||||
ordered = [c for c in popular_order if c in supported]
|
||||
ordered += sorted(c for c in supported if c not in ordered)
|
||||
|
||||
return {
|
||||
"supported_languages": {
|
||||
# Top 5 — dominant on the internet
|
||||
"en": "English",
|
||||
"es": "Spanish",
|
||||
"de": "German",
|
||||
"fr": "French",
|
||||
"ja": "Japanese",
|
||||
# Top 6-15
|
||||
"pt": "Portuguese",
|
||||
"ru": "Russian",
|
||||
"it": "Italian",
|
||||
"zh-CN": "Chinese (Simplified)",
|
||||
"zh-TW": "Chinese (Traditional)",
|
||||
"pl": "Polish",
|
||||
"nl": "Dutch",
|
||||
"tr": "Turkish",
|
||||
"ko": "Korean",
|
||||
"ar": "Arabic",
|
||||
# Top 16-25
|
||||
"fa": "Persian (Farsi)",
|
||||
"vi": "Vietnamese",
|
||||
"id": "Indonesian",
|
||||
"uk": "Ukrainian",
|
||||
"sv": "Swedish",
|
||||
"cs": "Czech",
|
||||
"el": "Greek",
|
||||
"he": "Hebrew",
|
||||
"hi": "Hindi",
|
||||
"ro": "Romanian",
|
||||
# Others
|
||||
"da": "Danish",
|
||||
"fi": "Finnish",
|
||||
"no": "Norwegian",
|
||||
"hu": "Hungarian",
|
||||
"th": "Thai",
|
||||
"sk": "Slovak",
|
||||
"bg": "Bulgarian",
|
||||
"hr": "Croatian",
|
||||
"ca": "Catalan",
|
||||
"ms": "Malay",
|
||||
},
|
||||
"supported_languages": {code: names.get(code, code.upper()) for code in ordered},
|
||||
"count": len(ordered),
|
||||
"note": "Supported languages may vary depending on the translation service configured",
|
||||
}
|
||||
|
||||
@@ -276,6 +280,10 @@ async def translate_batch_documents(
|
||||
|
||||
file_handler.save_upload_file(file, input_path)
|
||||
|
||||
# Zip bomb protection: Office files are ZIP archives
|
||||
if file_extension != ".pdf":
|
||||
validate_zip_safety(input_path)
|
||||
|
||||
if file_extension == ".xlsx":
|
||||
excel_translator.translate_file(
|
||||
input_path, output_path, target_language, source_language
|
||||
@@ -300,6 +308,18 @@ async def translate_batch_documents(
|
||||
}
|
||||
)
|
||||
|
||||
except ValueError as e:
|
||||
file_handler.cleanup_file(input_path)
|
||||
logger.warning(f"Rejected unsafe or invalid archive: {file.filename}: {e}")
|
||||
results.append(
|
||||
{
|
||||
"filename": file.filename,
|
||||
"status": "error",
|
||||
"error": "CORRUPTED_FILE",
|
||||
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
|
||||
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing {file.filename}")
|
||||
results.append(
|
||||
@@ -341,6 +361,9 @@ async def extract_texts_from_document(
|
||||
input_path = config.UPLOAD_DIR / input_filename
|
||||
file_handler.save_upload_file(file, input_path)
|
||||
|
||||
if file_extension != ".pdf":
|
||||
validate_zip_safety(input_path)
|
||||
|
||||
texts = []
|
||||
|
||||
if file_extension == ".xlsx":
|
||||
@@ -423,6 +446,17 @@ async def extract_texts_from_document(
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except ValueError as e:
|
||||
file_handler.cleanup_file(input_path)
|
||||
logger.warning(f"Text extraction rejected unsafe or invalid archive: {file.filename}: {e}")
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": "CORRUPTED_FILE",
|
||||
"message": "Le fichier semble corrompu ou n'est pas un document Office valide.",
|
||||
"details": {"reason": "unsafe_archive", "detail": str(e)[:200]},
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Text extraction error")
|
||||
return JSONResponse(
|
||||
|
||||
@@ -9,6 +9,7 @@ Story 3.6: Documentation OpenAPI complète avec exemples et codes d'erreur
|
||||
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
import time
|
||||
import socket
|
||||
@@ -43,7 +44,7 @@ from typing_extensions import Annotated
|
||||
|
||||
from config import config
|
||||
from translators import ExcelTranslator, WordTranslator, PowerPointTranslator
|
||||
from models.subscription import PlanType
|
||||
from models.subscription import PlanType, PLANS
|
||||
from services.auth_service import (
|
||||
record_usage,
|
||||
check_usage_limits,
|
||||
@@ -54,6 +55,7 @@ from middleware.tier_quota import _seconds_until_next_month, _next_month_utc
|
||||
from middleware.validation import FileValidator, ValidationError, LanguageValidator, webhook_validator
|
||||
from middleware.api_key_auth import get_authenticated_user, get_user_from_api_key
|
||||
from utils import file_handler
|
||||
from utils.file_handler import validate_zip_safety
|
||||
|
||||
# Import models from schemas (Story 3.6 - DRY principle)
|
||||
from schemas.translation import (
|
||||
@@ -148,6 +150,43 @@ def _tier_for_quota(plan) -> str:
|
||||
return "free"
|
||||
|
||||
|
||||
def _tier_from_plan_str(plan_str: Optional[str]) -> str:
|
||||
"""Map the job's ``user_plan`` string to a quota tier.
|
||||
|
||||
``user_plan`` is built with ``str(current_user.plan)`` which renders a
|
||||
str-Enum as "PlanType.PRO" — accept both that repr and the raw value.
|
||||
"""
|
||||
name = (plan_str or "").strip().lower()
|
||||
if name.startswith("plantype."):
|
||||
name = name.split(".", 1)[1]
|
||||
try:
|
||||
return _tier_for_quota(PlanType(name))
|
||||
except ValueError:
|
||||
return "free"
|
||||
|
||||
|
||||
def _plan_from_user(current_user: Optional[Any]) -> PlanType:
|
||||
"""Resolve the request's plan (anonymous requests are on the Free plan)."""
|
||||
if current_user is None:
|
||||
return PlanType.FREE
|
||||
plan = getattr(current_user, "plan", PlanType.FREE)
|
||||
try:
|
||||
return PlanType(plan)
|
||||
except ValueError:
|
||||
return PlanType.FREE
|
||||
|
||||
|
||||
def _allowed_providers_for_plan(plan: PlanType) -> set:
|
||||
"""Engines included in the plan (source of truth: PLANS[plan]["providers"])."""
|
||||
plan_cfg = PLANS.get(plan) or PLANS[PlanType.FREE]
|
||||
return set(plan_cfg.get("providers", []))
|
||||
|
||||
|
||||
def _image_translation_allowed_for_plan(plan: PlanType) -> bool:
|
||||
"""Vision (text-in-images) is sold on Pro and above."""
|
||||
return plan in (PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE)
|
||||
|
||||
|
||||
def _next_midnight_utc() -> datetime:
|
||||
"""Get next midnight UTC."""
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -215,6 +254,21 @@ def _parse_content_disposition(content_disp: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def _sanitize_url_filename(filename: str) -> str:
|
||||
"""Strip path traversal / control chars from a remote-provided filename."""
|
||||
filename = Path(filename).name
|
||||
filename = re.sub(r"[\x00-\x1f\x7f-\x9f]", "", filename)
|
||||
filename = re.sub(r'[<>:"/\\|?*]', "_", filename)
|
||||
# Drop leading dots (hidden files, ".." leftovers)
|
||||
filename = filename.lstrip(".")
|
||||
if len(filename) > 255:
|
||||
name, ext = (
|
||||
filename.rsplit(".", 1) if "." in filename else (filename, "")
|
||||
)
|
||||
filename = name[:250] + ("." + ext if ext else "")
|
||||
return filename or "downloaded_file"
|
||||
|
||||
|
||||
def _is_ssrf_risk(hostname: str) -> bool:
|
||||
"""Return True if hostname resolves to a private/reserved IP (SSRF prevention).
|
||||
|
||||
@@ -252,95 +306,130 @@ async def download_from_url(url: str, timeout: int = 30) -> tuple[Path, str]:
|
||||
details={"scheme": parsed_url.scheme or "none"},
|
||||
)
|
||||
|
||||
hostname = parsed_url.hostname or ""
|
||||
if not hostname or _is_ssrf_risk(hostname):
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message="The URL points to a blocked address (private or internal network).",
|
||||
details={"reason": "ssrf_blocked"},
|
||||
)
|
||||
def _validate_url_target(u: str):
|
||||
p = urlparse(u)
|
||||
if p.scheme not in ("http", "https"):
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message="Only HTTP/HTTPS URLs are accepted.",
|
||||
details={"scheme": p.scheme or "none"},
|
||||
)
|
||||
h = p.hostname or ""
|
||||
if not h or _is_ssrf_risk(h):
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message="The URL points to a blocked address (private or internal network).",
|
||||
details={"reason": "ssrf_blocked"},
|
||||
)
|
||||
|
||||
_validate_url_target(url)
|
||||
|
||||
MAX_REDIRECTS = 5
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout, follow_redirects=True, max_redirects=10
|
||||
timeout=timeout, follow_redirects=False
|
||||
) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
if response.status_code != 200:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message=f"URL unreachable (HTTP {response.status_code})",
|
||||
details={"status_code": response.status_code, "url": url[:100]},
|
||||
)
|
||||
|
||||
content_length = response.headers.get("content-length")
|
||||
if content_length:
|
||||
try:
|
||||
file_size = int(content_length)
|
||||
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
|
||||
if file_size > max_size_bytes:
|
||||
current_url = url
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
_validate_url_target(current_url)
|
||||
async with client.stream("GET", current_url) as response:
|
||||
if response.status_code in (301, 302, 303, 307, 308):
|
||||
location = response.headers.get("location", "")
|
||||
if not location:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.FILE_TOO_LARGE,
|
||||
message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
|
||||
details={
|
||||
"size_mb": round(file_size / (1024 * 1024), 2),
|
||||
"max_mb": MAX_FILE_SIZE_MB,
|
||||
},
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message="Redirect without Location header.",
|
||||
details={"reason": "invalid_redirect"},
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
# Re-check the redirect target before following it
|
||||
current_url = str(httpx.URL(current_url).join(location))
|
||||
continue
|
||||
|
||||
filename = None
|
||||
content_disp = response.headers.get("content-disposition", "")
|
||||
if content_disp:
|
||||
filename = _parse_content_disposition(content_disp)
|
||||
if response.status_code != 200:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message=f"URL unreachable (HTTP {response.status_code})",
|
||||
details={"status_code": response.status_code, "url": url[:100]},
|
||||
)
|
||||
|
||||
if not filename:
|
||||
filename = unquote(Path(parsed_url.path).name) or "downloaded_file"
|
||||
content_length = response.headers.get("content-length")
|
||||
if content_length:
|
||||
try:
|
||||
file_size = int(content_length)
|
||||
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
|
||||
if file_size > max_size_bytes:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.FILE_TOO_LARGE,
|
||||
message=f"File is too large ({round(file_size / (1024 * 1024), 2)} MB, max {MAX_FILE_SIZE_MB} MB).",
|
||||
details={
|
||||
"size_mb": round(file_size / (1024 * 1024), 2),
|
||||
"max_mb": MAX_FILE_SIZE_MB,
|
||||
},
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
extension = Path(filename).suffix.lower()
|
||||
if extension not in ACCEPTED_EXTENSIONS:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.INVALID_FORMAT,
|
||||
details={
|
||||
"detected_extension": extension or "none",
|
||||
"accepted_formats": list(ACCEPTED_EXTENSIONS),
|
||||
},
|
||||
)
|
||||
filename = None
|
||||
content_disp = response.headers.get("content-disposition", "")
|
||||
if content_disp:
|
||||
filename = _parse_content_disposition(content_disp)
|
||||
|
||||
unique_id = str(uuid.uuid4())[:8]
|
||||
safe_filename = f"{unique_id}_{filename}"
|
||||
temp_path = config.UPLOAD_DIR / safe_filename
|
||||
if not filename:
|
||||
filename = unquote(Path(urlparse(current_url).path).name)
|
||||
|
||||
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
filename = _sanitize_url_filename(filename)
|
||||
|
||||
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
|
||||
downloaded_bytes = 0
|
||||
extension = Path(filename).suffix.lower()
|
||||
if extension not in ACCEPTED_EXTENSIONS:
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.INVALID_FORMAT,
|
||||
details={
|
||||
"detected_extension": extension or "none",
|
||||
"accepted_formats": list(ACCEPTED_EXTENSIONS),
|
||||
},
|
||||
)
|
||||
|
||||
async with aiofiles.open(temp_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=65536):
|
||||
downloaded_bytes += len(chunk)
|
||||
unique_id = str(uuid.uuid4())[:8]
|
||||
safe_filename = f"{unique_id}_{filename}"
|
||||
temp_path = config.UPLOAD_DIR / safe_filename
|
||||
|
||||
if downloaded_bytes > max_size_bytes:
|
||||
await f.close()
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.FILE_TOO_LARGE,
|
||||
details={
|
||||
"size_mb": round(
|
||||
downloaded_bytes / (1024 * 1024), 2
|
||||
),
|
||||
"max_mb": MAX_FILE_SIZE_MB,
|
||||
},
|
||||
)
|
||||
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
await f.write(chunk)
|
||||
max_size_bytes = MAX_FILE_SIZE_MB * 1024 * 1024
|
||||
downloaded_bytes = 0
|
||||
|
||||
async with aiofiles.open(temp_path, "rb") as f:
|
||||
header = await f.read(4)
|
||||
await validate_file_content(header, extension)
|
||||
async with aiofiles.open(temp_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=65536):
|
||||
downloaded_bytes += len(chunk)
|
||||
|
||||
return temp_path, filename
|
||||
if downloaded_bytes > max_size_bytes:
|
||||
await f.close()
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.FILE_TOO_LARGE,
|
||||
details={
|
||||
"size_mb": round(
|
||||
downloaded_bytes / (1024 * 1024), 2
|
||||
),
|
||||
"max_mb": MAX_FILE_SIZE_MB,
|
||||
},
|
||||
)
|
||||
|
||||
await f.write(chunk)
|
||||
|
||||
async with aiofiles.open(temp_path, "rb") as f:
|
||||
header = await f.read(4)
|
||||
await validate_file_content(header, extension)
|
||||
|
||||
return temp_path, filename
|
||||
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.URL_UNREACHABLE,
|
||||
message="Too many redirects.",
|
||||
details={"reason": "too_many_redirects", "max_redirects": MAX_REDIRECTS},
|
||||
)
|
||||
|
||||
except httpx.TimeoutException:
|
||||
if temp_path and temp_path.exists():
|
||||
@@ -420,9 +509,13 @@ def _cleanup_old_jobs() -> None:
|
||||
return
|
||||
_last_cleanup_ts = current_time
|
||||
|
||||
# Snapshot the items before filtering: concurrent coroutines (background
|
||||
# translation workers, status pollers) mutate _translation_jobs, and
|
||||
# iterating a dict while it is resized raises
|
||||
# "RuntimeError: dictionary changed size during iteration".
|
||||
expired_job_ids = [
|
||||
job_id
|
||||
for job_id, job in _translation_jobs.items()
|
||||
for job_id, job in list(_translation_jobs.items())
|
||||
if job.get("status") in ("completed", "failed")
|
||||
and (
|
||||
(ts := job.get("completed_at") or job.get("failed_at"))
|
||||
@@ -444,6 +537,68 @@ def _job_age_seconds(timestamp_str: str) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _provider_model(provider: Any) -> str:
|
||||
"""Best-effort read of a provider's model name.
|
||||
|
||||
The new-style providers (``services/providers/*``) store the model in
|
||||
``self._model``, while the legacy providers (``services/translation_service``)
|
||||
expose ``self.model``. Read either so cost-factor logic works for both.
|
||||
"""
|
||||
if provider is None:
|
||||
return ""
|
||||
return (
|
||||
getattr(provider, "_model", None)
|
||||
or getattr(provider, "model", None)
|
||||
or ""
|
||||
)
|
||||
|
||||
|
||||
def _compute_cost_factor(provider: Any, provider_name: str = "") -> int:
|
||||
"""Billing cost factor (1 = standard, 5 = premium).
|
||||
|
||||
Premium models (Claude, GPT-4 family, etc.) cost more and are billed at a
|
||||
higher factor. Cheap variants are explicitly downgraded to 1 (``haiku``,
|
||||
and the small GPT-4 models such as ``gpt-4o-mini`` / ``gpt-4o-nano``).
|
||||
The provider may be passed by instance (model read off it) or only by
|
||||
name (e.g. the ``openrouter_premium`` alias).
|
||||
"""
|
||||
model_lower = _provider_model(provider).lower()
|
||||
provider_lower = (provider_name or "").lower()
|
||||
|
||||
if "haiku" in model_lower or "mini" in model_lower or "nano" in model_lower:
|
||||
return 1
|
||||
if any(k in model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
|
||||
return 5
|
||||
return 1
|
||||
|
||||
|
||||
def _compute_duration_seconds(created_at_iso: str) -> float:
|
||||
"""Elapsed seconds since ``created_at`` (UTC ISO string).
|
||||
|
||||
Uses a timezone-aware timestamp() to avoid the local-time bug that
|
||||
``time.mktime`` introduced. Falls back to 0 on parse errors so a bad
|
||||
timestamp can never flip a successful job into the error branch.
|
||||
"""
|
||||
return _job_age_seconds(created_at_iso)
|
||||
|
||||
|
||||
async def _release_quota_if_needed(user_id, usage_recorded: bool, job_id: str) -> None:
|
||||
"""Release the reserved translation quota on a soft-failure path.
|
||||
|
||||
The translation worker reserves a quota slot at request time. The generic
|
||||
``except`` branch releases it on hard failures, but the early ``return``
|
||||
paths (empty output / no translatable text / 0 texts translated) are NOT
|
||||
exceptions and must release the quota explicitly — otherwise the user
|
||||
loses a slot without receiving a translation.
|
||||
"""
|
||||
if user_id and not usage_recorded:
|
||||
try:
|
||||
await asyncio.to_thread(release_translation_quota, user_id)
|
||||
logger.info(f"Job {job_id}: released reserved quota after soft-failure")
|
||||
except Exception as release_err:
|
||||
logger.exception(f"Job {job_id}: failed to release reserved quota: {release_err}")
|
||||
|
||||
|
||||
@router_v1.post(
|
||||
"/translate",
|
||||
response_model=TranslateResponse,
|
||||
@@ -476,6 +631,14 @@ async def translate_document_v1(
|
||||
pdf_mode: Optional[Literal["layout", "text_only"]] = Form(
|
||||
default=None, description="PDF translation mode: 'layout' (preserve layout) or 'text_only' (clean text output). PDF only."
|
||||
),
|
||||
formality: Optional[Literal["formal", "informal"]] = Form(
|
||||
default=None,
|
||||
description="Tone override for LLM engines: 'formal' or 'informal'. Ignored by classic engines.",
|
||||
),
|
||||
output_mode: Optional[Literal["single", "bilingual"]] = Form(
|
||||
default="single",
|
||||
description="Output mode: 'single' (translated file only) or 'bilingual' (docx with source + translation interleaved).",
|
||||
),
|
||||
translate_images: bool = Form(
|
||||
default=False, description="Translate text inside images using AI vision"
|
||||
),
|
||||
@@ -676,7 +839,9 @@ async def translate_document_v1(
|
||||
rate_limit_remaining = -1
|
||||
|
||||
try:
|
||||
LanguageValidator.validate(target_lang)
|
||||
# Keep the canonical form (e.g. "zh-CN") so providers receive a
|
||||
# code they understand instead of the raw user input.
|
||||
target_lang = LanguageValidator.validate(target_lang)
|
||||
except ValidationError as e:
|
||||
raise TranslateEndpointError(
|
||||
code="INVALID_FORMAT",
|
||||
@@ -686,7 +851,7 @@ async def translate_document_v1(
|
||||
|
||||
if source_lang and source_lang != "auto":
|
||||
try:
|
||||
LanguageValidator.validate(source_lang)
|
||||
source_lang = LanguageValidator.validate(source_lang)
|
||||
except ValidationError:
|
||||
raise TranslateEndpointError(
|
||||
code="INVALID_FORMAT",
|
||||
@@ -760,7 +925,23 @@ async def translate_document_v1(
|
||||
details={"error": "sha256_calculation_failed"},
|
||||
)
|
||||
|
||||
# Office files are ZIP archives: reject archives that expand far
|
||||
# beyond their uploaded size (zip bomb protection).
|
||||
if file_extension != ".pdf":
|
||||
try:
|
||||
validate_zip_safety(input_path)
|
||||
except ValueError as e:
|
||||
file_handler_util.cleanup_file(input_path)
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.CORRUPTED_FILE,
|
||||
message="The file is invalid or expands dangerously.",
|
||||
details={"reason": "unsafe_archive", "detail": str(e)[:200]},
|
||||
)
|
||||
|
||||
job_id = f"tr_{uuid.uuid4().hex[:12]}"
|
||||
# Secret per-job token: required to follow/download a job that has no
|
||||
# logged-in owner (anonymous API calls).
|
||||
job_access_token = secrets.token_urlsafe(24)
|
||||
|
||||
# Track file metadata in Redis with TTL
|
||||
await storage_tracker.track_file(
|
||||
@@ -771,6 +952,7 @@ async def translate_document_v1(
|
||||
"file_hash": file_hash,
|
||||
"input_path": str(input_path),
|
||||
"user_id": str(user_id) if user_id else None,
|
||||
"access_token": job_access_token,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
@@ -794,6 +976,7 @@ async def translate_document_v1(
|
||||
"target_lang": target_lang,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"user_id": user_id,
|
||||
"access_token": job_access_token,
|
||||
"input_path": str(input_path),
|
||||
"file_extension": file_extension,
|
||||
"provider": provider or mode,
|
||||
@@ -803,6 +986,8 @@ async def translate_document_v1(
|
||||
"prompt_id": prompt_id, # Story 3.12: Store prompt_id
|
||||
"pdf_mode": pdf_mode, # PDF translation mode
|
||||
"translate_images": translate_images,
|
||||
"formality": formality, # LLM tone override
|
||||
"output_mode": output_mode or "single", # single | bilingual
|
||||
}
|
||||
await set_job_status_async(job_id, _translation_jobs[job_id])
|
||||
|
||||
@@ -821,6 +1006,39 @@ async def translate_document_v1(
|
||||
)
|
||||
provider_to_use = "google"
|
||||
|
||||
# ── Plan-based engine gating (grille commerciale : PLANS[plan]["providers"]) ──
|
||||
# Chaque plan n'expose que ses moteurs ; un moteur hors plan est refusé
|
||||
# (403) au lieu d'être exécuté aux frais de la maison.
|
||||
_user_plan = _plan_from_user(current_user)
|
||||
if provider_to_use not in _allowed_providers_for_plan(_user_plan):
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
|
||||
message=(
|
||||
f"Le moteur « {provider_to_use} » n'est pas inclus dans votre plan. "
|
||||
"Passez à un plan supérieur pour l'utiliser."
|
||||
),
|
||||
details={
|
||||
"feature": "provider",
|
||||
"provider": provider_to_use,
|
||||
"plan": str(_user_plan.value),
|
||||
"allowed_providers": sorted(_allowed_providers_for_plan(_user_plan)),
|
||||
},
|
||||
)
|
||||
|
||||
# La traduction du texte dans les images (vision) est vendue Pro et plus.
|
||||
if translate_images and not _image_translation_allowed_for_plan(_user_plan):
|
||||
raise TranslateEndpointError(
|
||||
code=TranslateEndpointError.PRO_FEATURE_REQUIRED,
|
||||
message=(
|
||||
"La traduction du texte dans les images est réservée "
|
||||
"aux plans Pro et supérieurs."
|
||||
),
|
||||
details={
|
||||
"feature": "translate_images",
|
||||
"plan": str(_user_plan.value),
|
||||
},
|
||||
)
|
||||
|
||||
asyncio.create_task(
|
||||
_run_translation_job(
|
||||
job_id=job_id,
|
||||
@@ -837,6 +1055,8 @@ async def translate_document_v1(
|
||||
user_plan=str(current_user.plan) if current_user else "free",
|
||||
pdf_mode=pdf_mode,
|
||||
translate_images=translate_images,
|
||||
formality=formality,
|
||||
output_mode=output_mode or "single",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -853,6 +1073,8 @@ async def translate_document_v1(
|
||||
"file_name": original_filename,
|
||||
"source_lang": source_lang,
|
||||
"target_lang": target_lang,
|
||||
# Needed to poll/download this job when calling without login
|
||||
"access_token": job_access_token,
|
||||
},
|
||||
"meta": {
|
||||
"rate_limit_remaining": rate_limit_remaining,
|
||||
@@ -980,6 +1202,8 @@ async def _run_translation_job(
|
||||
user_plan: Optional[str] = None, # Plan name for watermark decision
|
||||
pdf_mode: Optional[str] = None, # PDF translation mode: "layout" or "text_only"
|
||||
translate_images: bool = False,
|
||||
formality: Optional[str] = None, # LLM tone override: formal/informal
|
||||
output_mode: str = "single", # single | bilingual
|
||||
) -> None:
|
||||
"""
|
||||
Run translation job in background with progress tracking.
|
||||
@@ -1076,11 +1300,13 @@ async def _run_translation_job(
|
||||
# Use custom_prompt if no prompt_id
|
||||
effective_prompt = custom_prompt
|
||||
|
||||
# Build the full prompt combining effective prompt and glossary
|
||||
# Build the full prompt combining effective prompt, glossary,
|
||||
# formality directive and regional variant hint.
|
||||
full_prompt = build_full_prompt(
|
||||
effective_prompt, glossary_terms,
|
||||
source_lang=glossary_source_lang, target_lang=target_lang,
|
||||
glossary_target_lang=glossary_target_lang,
|
||||
formality=formality,
|
||||
)
|
||||
|
||||
from services.providers.google_provider import GoogleTranslationProvider
|
||||
@@ -1259,6 +1485,8 @@ async def _run_translation_job(
|
||||
job_translator = ExcelTranslator(provider=translation_provider)
|
||||
if hasattr(job_translator, "set_custom_prompt"):
|
||||
job_translator.set_custom_prompt(full_prompt)
|
||||
if hasattr(job_translator, "set_tm_scope"):
|
||||
job_translator.set_tm_scope(user_id, full_prompt)
|
||||
await asyncio.to_thread(
|
||||
job_translator.translate_file,
|
||||
input_path,
|
||||
@@ -1272,6 +1500,8 @@ async def _run_translation_job(
|
||||
job_translator = WordTranslator(provider=translation_provider)
|
||||
if hasattr(job_translator, "set_custom_prompt"):
|
||||
job_translator.set_custom_prompt(full_prompt)
|
||||
if hasattr(job_translator, "set_tm_scope"):
|
||||
job_translator.set_tm_scope(user_id, full_prompt)
|
||||
await asyncio.to_thread(
|
||||
job_translator.translate_file,
|
||||
input_path,
|
||||
@@ -1285,6 +1515,8 @@ async def _run_translation_job(
|
||||
job_translator = PowerPointTranslator(provider=translation_provider)
|
||||
if hasattr(job_translator, "set_custom_prompt"):
|
||||
job_translator.set_custom_prompt(full_prompt)
|
||||
if hasattr(job_translator, "set_tm_scope"):
|
||||
job_translator.set_tm_scope(user_id, full_prompt)
|
||||
await asyncio.to_thread(
|
||||
job_translator.translate_file,
|
||||
input_path,
|
||||
@@ -1299,6 +1531,40 @@ async def _run_translation_job(
|
||||
job_translator = PDFTranslator(provider=translation_provider)
|
||||
if hasattr(job_translator, "set_custom_prompt"):
|
||||
job_translator.set_custom_prompt(full_prompt)
|
||||
if hasattr(job_translator, "set_tm_scope"):
|
||||
job_translator.set_tm_scope(user_id, full_prompt)
|
||||
# OCR (PDF scannés) : réglages admin > variables d'env.
|
||||
mistral_cfg = getattr(_admin_cfg, "mistral", None)
|
||||
mistral_key = _cfg(
|
||||
getattr(mistral_cfg, "api_key", None), "MISTRAL_API_KEY"
|
||||
)
|
||||
# Only honor the admin "enabled" flag when Mistral is actually
|
||||
# configured there — a freshly saved settings file carries
|
||||
# enabled=false defaults that must not override env-based OCR.
|
||||
mistral_admin_configured = bool(
|
||||
mistral_cfg is not None
|
||||
and (getattr(mistral_cfg, "api_key", None) or "").strip()
|
||||
)
|
||||
job_translator.set_ocr_config(
|
||||
api_key=mistral_key,
|
||||
model=_cfg(
|
||||
getattr(mistral_cfg, "model", None),
|
||||
"MISTRAL_OCR_MODEL",
|
||||
"mistral-ocr-latest",
|
||||
),
|
||||
timeout=int(
|
||||
_cfg(
|
||||
str(getattr(mistral_cfg, "timeout", "") or ""),
|
||||
"MISTRAL_OCR_TIMEOUT",
|
||||
"180",
|
||||
)
|
||||
),
|
||||
enabled=(
|
||||
getattr(mistral_cfg, "enabled", None)
|
||||
if mistral_admin_configured
|
||||
else None
|
||||
),
|
||||
)
|
||||
actual_output = await asyncio.to_thread(
|
||||
job_translator.translate_file,
|
||||
input_path,
|
||||
@@ -1320,19 +1586,21 @@ async def _run_translation_job(
|
||||
error_msg = "Translation failed: output file is empty or missing. The translation provider may be unavailable."
|
||||
logger.error(f"Job {job_id}: {error_msg}")
|
||||
tracker.set_error(error_msg)
|
||||
await _release_quota_if_needed(user_id, usage_recorded, job_id)
|
||||
return
|
||||
|
||||
stats = job_translator.get_translation_stats()
|
||||
attempted = stats.get("attempted", 0)
|
||||
changed = stats.get("changed", 0)
|
||||
|
||||
if attempted == 0 and file_extension in ('.docx', '.xlsx', '.pptx'):
|
||||
if attempted == 0:
|
||||
error_msg = (
|
||||
"Aucun texte traduisible détecté dans le document. "
|
||||
"Le fichier est peut-être vide, protégé, ou ne contient que des images."
|
||||
)
|
||||
logger.error(f"Job {job_id}: {error_msg}")
|
||||
tracker.set_error(error_msg)
|
||||
await _release_quota_if_needed(user_id, usage_recorded, job_id)
|
||||
return
|
||||
|
||||
if attempted > 0:
|
||||
@@ -1346,6 +1614,7 @@ async def _run_translation_job(
|
||||
)
|
||||
logger.error(f"Job {job_id}: {error_msg}")
|
||||
tracker.set_error(error_msg)
|
||||
await _release_quota_if_needed(user_id, usage_recorded, job_id)
|
||||
return
|
||||
elif ratio < 0.05:
|
||||
# Very suspicious — likely partial failure, warn but don't block
|
||||
@@ -1430,9 +1699,7 @@ async def _run_translation_job(
|
||||
from middleware.metrics import record_translation_retry
|
||||
record_translation_retry(
|
||||
reason="l1_fail",
|
||||
tier=_tier_for_quota(
|
||||
current_user.plan if current_user else None
|
||||
) if current_user else "free",
|
||||
tier=_tier_from_plan_str(user_plan),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1448,9 +1715,7 @@ async def _run_translation_job(
|
||||
from middleware.metrics import record_translation_retry
|
||||
record_translation_retry(
|
||||
reason="l0_fail",
|
||||
tier=_tier_for_quota(
|
||||
current_user.plan if current_user else None
|
||||
) if current_user else "free",
|
||||
tier=_tier_from_plan_str(user_plan),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1467,9 +1732,7 @@ async def _run_translation_job(
|
||||
from services.quality import run_l2_check
|
||||
|
||||
# Tier gate: Pro+ plans only (unless gate is disabled)
|
||||
user_tier = (
|
||||
_tier_for_quota(current_user.plan) if current_user else "free"
|
||||
)
|
||||
user_tier = _tier_from_plan_str(user_plan)
|
||||
tier_gate_on = getattr(config, "QUALITY_L2_TIER_GATE", True)
|
||||
if not tier_gate_on or user_tier in ("pro", "business", "enterprise"):
|
||||
translated_chunks_for_l2 = [s["translated"] for s in quality_samples]
|
||||
@@ -1498,21 +1761,52 @@ async def _run_translation_job(
|
||||
f"Job {job_id}: quality L2 layer failed: {l2_err}"
|
||||
)
|
||||
|
||||
if user_id:
|
||||
# Determine cost factor based on selected provider and model
|
||||
cost_factor = 1
|
||||
provider_lower = (provider or "").lower()
|
||||
|
||||
prov_model = ""
|
||||
if translation_provider:
|
||||
prov_model = getattr(translation_provider, "model", "") or ""
|
||||
|
||||
prov_model_lower = prov_model.lower()
|
||||
if any(k in prov_model_lower for k in ["claude", "fable", "gpt-4"]) or provider_lower == "openrouter_premium":
|
||||
if "haiku" in prov_model_lower:
|
||||
cost_factor = 1
|
||||
# ------------------------------------------------------------------
|
||||
# QA report (pure heuristics — numbers fidelity, untranslated
|
||||
# ratio, 0-100 score). Never blocks the job; surfaced in the job
|
||||
# status for the UI/API.
|
||||
# ------------------------------------------------------------------
|
||||
try:
|
||||
from services.quality.qa_report import run_qa_report
|
||||
|
||||
qa = await asyncio.to_thread(
|
||||
run_qa_report, input_path, output_path, target_lang, file_extension
|
||||
)
|
||||
if qa:
|
||||
job["quality"] = qa
|
||||
except Exception as qa_err:
|
||||
logger.warning(f"Job {job_id}: QA report failed: {qa_err}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Bilingual output (docx): interleaves source paragraphs above
|
||||
# their translation. Falls back silently to the translated file.
|
||||
# ------------------------------------------------------------------
|
||||
if output_mode == "bilingual" and file_extension == ".docx":
|
||||
try:
|
||||
from translators.bilingual import make_bilingual_docx
|
||||
|
||||
bilingual_path = output_path.with_name(
|
||||
output_path.stem + "_bilingual" + output_path.suffix
|
||||
)
|
||||
actual = await asyncio.to_thread(
|
||||
make_bilingual_docx, input_path, output_path, bilingual_path
|
||||
)
|
||||
if actual:
|
||||
output_path = Path(actual)
|
||||
logger.info(f"Job {job_id}: bilingual output generated")
|
||||
else:
|
||||
cost_factor = 5
|
||||
logger.warning(
|
||||
f"Job {job_id}: bilingual output skipped "
|
||||
"(structure mismatch) — returning translated file"
|
||||
)
|
||||
except Exception as bi_err:
|
||||
logger.warning(f"Job {job_id}: bilingual output failed: {bi_err}")
|
||||
|
||||
if user_id:
|
||||
# Determine cost factor based on selected provider and model.
|
||||
# _compute_cost_factor reads the model off the provider robustly
|
||||
# (new-style providers store it in ``_model``, legacy in ``model``).
|
||||
cost_factor = _compute_cost_factor(translation_provider, provider or "")
|
||||
|
||||
# Persist monthly usage counters in PostgreSQL (docs + pages)
|
||||
pages = await asyncio.to_thread(
|
||||
@@ -1537,7 +1831,7 @@ async def _run_translation_job(
|
||||
|
||||
tracker.set_completed(str(output_path))
|
||||
# Record translation metric
|
||||
duration = time.time() - time.mktime(datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).timetuple())
|
||||
duration = _compute_duration_seconds(job.get("created_at", ""))
|
||||
record_translation(provider=provider, file_type=file_extension or "unknown", duration=duration, status="success")
|
||||
logger.info(f"Job {job_id}: Completed successfully")
|
||||
|
||||
@@ -1621,6 +1915,50 @@ async def _run_translation_job(
|
||||
)
|
||||
|
||||
|
||||
def _check_job_access(
|
||||
job: dict, current_user, token: Optional[str]
|
||||
) -> Optional[JSONResponse]:
|
||||
"""Return an error response if the caller may not see this job, else None.
|
||||
|
||||
Jobs owned by a logged-in user require that same user. Anonymous jobs
|
||||
require the secret per-job token returned when the job was created.
|
||||
"""
|
||||
job_user_id = job.get("user_id")
|
||||
if job_user_id:
|
||||
if not current_user:
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={
|
||||
"error": "AUTH_REQUIRED",
|
||||
"message": "Authentication is required to access this job.",
|
||||
"details": {"job_id": job.get("id")},
|
||||
},
|
||||
)
|
||||
if str(job_user_id) != str(current_user.id):
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={
|
||||
"error": "ACCESS_DENIED",
|
||||
"message": "You do not have access to this job.",
|
||||
"details": {"job_id": job.get("id")},
|
||||
},
|
||||
)
|
||||
return None
|
||||
|
||||
expected = job.get("access_token")
|
||||
provided = token or ""
|
||||
if not expected or not secrets.compare_digest(provided, expected):
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={
|
||||
"error": "ACCESS_DENIED",
|
||||
"message": "You do not have access to this job.",
|
||||
"details": {"job_id": job.get("id"), "hint": "token_required"},
|
||||
},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@router_v1.get(
|
||||
"/translations/{job_id}",
|
||||
response_model=TranslationStatusResponse,
|
||||
@@ -1631,6 +1969,7 @@ async def _run_translation_job(
|
||||
)
|
||||
async def get_translation_status(
|
||||
job_id: str,
|
||||
token: Optional[str] = None,
|
||||
current_user: Optional[Any] = Depends(get_authenticated_user),
|
||||
):
|
||||
"""
|
||||
@@ -1680,6 +2019,10 @@ async def get_translation_status(
|
||||
},
|
||||
)
|
||||
|
||||
denied = _check_job_access(job, current_user, token)
|
||||
if denied:
|
||||
return denied
|
||||
|
||||
response_data = {
|
||||
"id": job["id"],
|
||||
"status": job["status"],
|
||||
@@ -1689,6 +2032,7 @@ async def get_translation_status(
|
||||
"source_lang": job.get("source_lang"),
|
||||
"target_lang": job.get("target_lang"),
|
||||
"created_at": job.get("created_at"),
|
||||
"quality": job.get("quality"),
|
||||
}
|
||||
|
||||
estimated_remaining = None
|
||||
@@ -1768,6 +2112,7 @@ def _cleanup_files(input_path: Optional[str], output_path: Optional[str]) -> Non
|
||||
)
|
||||
async def download_translated_file(
|
||||
job_id: str,
|
||||
token: Optional[str] = None,
|
||||
current_user: Optional[Any] = Depends(get_authenticated_user),
|
||||
):
|
||||
"""
|
||||
@@ -1819,16 +2164,9 @@ async def download_translated_file(
|
||||
},
|
||||
)
|
||||
|
||||
job_user_id = job.get("user_id")
|
||||
if current_user and job_user_id and str(job_user_id) != str(current_user.id):
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={
|
||||
"error": "ACCESS_DENIED",
|
||||
"message": "You do not have access to this file.",
|
||||
"details": {"job_id": job_id},
|
||||
},
|
||||
)
|
||||
denied = _check_job_access(job, current_user, token)
|
||||
if denied:
|
||||
return denied
|
||||
|
||||
if job.get("status") != "completed":
|
||||
return JSONResponse(
|
||||
|
||||
98
routes/waitlist_routes.py
Normal file
98
routes/waitlist_routes.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
Waitlist / email-capture routes for the marketing landing page.
|
||||
|
||||
Public endpoint: POST /api/v1/waitlist (no auth) — deduplicated by email,
|
||||
persisted in data/waitlist.json (same JSON-file pattern as provider settings).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, EmailStr
|
||||
|
||||
from config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/waitlist", tags=["Waitlist"])
|
||||
|
||||
WAITLIST_FILE = config.BASE_DIR / "data" / "waitlist.json"
|
||||
_write_lock = threading.Lock()
|
||||
|
||||
|
||||
class WaitlistEntry(BaseModel):
|
||||
email: EmailStr
|
||||
interest: Optional[str] = None
|
||||
|
||||
|
||||
def _load_waitlist() -> List[dict]:
|
||||
if not WAITLIST_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
with open(WAITLIST_FILE, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load waitlist: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _save_waitlist(entries: List[dict]) -> None:
|
||||
WAITLIST_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(WAITLIST_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(entries, f, indent=2)
|
||||
|
||||
|
||||
@router.post("", status_code=201, summary="Join the waitlist")
|
||||
async def join_waitlist(entry: WaitlistEntry):
|
||||
email = entry.email.lower().strip()
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
with _write_lock:
|
||||
entries = _load_waitlist()
|
||||
for existing in entries:
|
||||
if existing.get("email", "").lower() == email:
|
||||
existing["updated_at"] = now
|
||||
if entry.interest:
|
||||
existing["interest"] = entry.interest
|
||||
_save_waitlist(entries)
|
||||
logger.info(f"Waitlist rejoin: {email}")
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={
|
||||
"data": {"email": email, "status": "already_joined"},
|
||||
"message": "You are already on the list.",
|
||||
},
|
||||
)
|
||||
|
||||
entries.append(
|
||||
{
|
||||
"email": email,
|
||||
"interest": entry.interest,
|
||||
"joined_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
_save_waitlist(entries)
|
||||
logger.info(f"Waitlist join: {email} (total={len(entries)})")
|
||||
|
||||
return JSONResponse(
|
||||
status_code=201,
|
||||
content={
|
||||
"data": {"email": email, "status": "joined"},
|
||||
"message": "Welcome aboard!",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/count", summary="Waitlist count")
|
||||
async def waitlist_count():
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={"data": {"count": len(_load_waitlist())}},
|
||||
)
|
||||
Reference in New Issue
Block a user