feat: production deployment - full update with providers, admin, glossaries, pricing, tests

Major changes across backend, frontend, infrastructure: - Provider system with model selection (Google, DeepL, OpenAI, Ollama, Google Cloud) - Admin panel: user management, pricing, settings - Glossary system with CSV import/export - Subscription and tier quota management - Security hardening (rate limiting, API key auth, path traversal fixes) - Docker compose for dev, prod, and IONOS deployment - Alembic migrations for new tables - Frontend: dashboard, pricing page, landing page, i18n (en/fr) - Test suite and verification scripts Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-04-25 15:01:47 +02:00
parent 2ba4fedfc8
commit 26bd096a06
1178 changed files with 136435 additions and 3047 deletions
--- a/routes/translate_routes.py
+++ b/routes/translate_routes.py
@@ -44,6 +44,7 @@ from typing_extensions import Annotated
 from config import config
 from models.subscription import PlanType
 from middleware.tier_quota import tier_quota_service
+from services.auth_service import record_usage
 from middleware.validation import FileValidator, ValidationError, LanguageValidator, webhook_validator
 from middleware.api_key_auth import get_authenticated_user, get_user_from_api_key
 from utils import file_handler
@@ -61,6 +62,7 @@ from schemas.errors import ErrorResponse
 from utils.file_handler import FileHandler
 from services.progress_tracker import ProgressTracker
 from services.storage_tracker import storage_tracker
+from core.redis import set_job_status_async, get_job_status_async
 from services.glossary_service import get_glossary_terms, validate_glossary_access, build_full_prompt
 from services.prompt_service import get_prompt_content, validate_prompt_access
 from utils.exceptions import GlossaryNotFoundError, PromptNotFoundError
@@ -700,9 +702,23 @@ async def translate_document_v1(
            "glossary_id": glossary_id,
            "prompt_id": prompt_id,  # Story 3.12: Store prompt_id
        }
+        await set_job_status_async(job_id, _translation_jobs[job_id])

        provider_to_use = provider or ("openrouter" if mode == "llm" else "google")

+        # google_cloud (API officielle payante) est réservé aux forfaits Pro et supérieurs.
+        # Les plans free/starter sont silencieusement redirigés vers le Google Translate gratuit.
+        if provider_to_use == "google_cloud":
+            _paid_plans = (PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE)
+            if not current_user or current_user.plan not in _paid_plans:
+                logger.info(
+                    "google_cloud_downgraded_to_google",
+                    reason="plan_restriction",
+                    plan=str(current_user.plan) if current_user else "anonymous",
+                    user_id=str(current_user.id) if current_user else None,
+                )
+                provider_to_use = "google"
+
        asyncio.create_task(
            _run_translation_job(
                job_id=job_id,
@@ -767,6 +783,36 @@ async def translate_document_v1(
        )


+def _estimate_pages(file_path: Path, file_extension: str) -> int:
+    """
+    Lightweight page-count estimate for usage accounting.
+    - .pptx : number of slides (exact)
+    - .xlsx : number of visible sheets
+    - .docx : rough estimate (~2 500 chars per page)
+    Returns at least 1.
+    """
+    try:
+        ext = file_extension.lower()
+        if ext == ".pptx":
+            from pptx import Presentation  # already a dep
+            prs = Presentation(str(file_path))
+            return max(1, len(prs.slides))
+        elif ext == ".xlsx":
+            import openpyxl  # already a dep
+            wb = openpyxl.load_workbook(str(file_path), read_only=True, data_only=True)
+            count = len(wb.sheetnames)
+            wb.close()
+            return max(1, count)
+        elif ext == ".docx":
+            import docx  # already a dep
+            doc = docx.Document(str(file_path))
+            char_count = sum(len(p.text) for p in doc.paragraphs)
+            return max(1, round(char_count / 2500))
+    except Exception as exc:
+        logger.warning(f"_estimate_pages failed for {file_extension}: {exc}")
+    return 1
+
+
 async def _run_translation_job(
    job_id: str,
    input_path: Path,
@@ -804,14 +850,28 @@ async def _run_translation_job(

    try:
        job["status"] = "processing"
+        await set_job_status_async(job_id, dict(job))
        tracker.update(10, "Validating file")

+        async def _sync_job_to_redis():
+            """Sync job status to Redis every 0.5s until completed/failed or job removed."""
+            while True:
+                await asyncio.sleep(0.5)
+                j = _translation_jobs.get(job_id)
+                if not j:
+                    break
+                await set_job_status_async(job_id, dict(j))
+                if j.get("status") in ("completed", "failed"):
+                    break
+
+        asyncio.create_task(_sync_job_to_redis())
+
        output_filename = file_handler_util.generate_unique_filename(
            input_path.name.replace("input_", "translated_"), "translated"
        )
        output_path = config.OUTPUT_DIR / output_filename

-        from translators import excel_translator, word_translator, pptx_translator
+        from translators import ExcelTranslator, WordTranslator, PowerPointTranslator
        from services.translation_service import (
            OpenRouterTranslationProvider,
            OllamaTranslationProvider,
@@ -887,7 +947,7 @@ async def _run_translation_job(
            deepl_key = _cfg(_admin_cfg.deepl.api_key, "DEEPL_API_KEY")
            if deepl_key:
                from services.translation_service import DeepLTranslationProvider
-                translation_provider = DeepLTranslationProvider(deepl_key, full_prompt)
+                translation_provider = DeepLTranslationProvider(deepl_key)
        elif _p == "zai":
            from services.translation_service import OpenAITranslationProvider as _OAI
            zai_key = _cfg(_admin_cfg.zai.api_key, "ZAI_API_KEY")
@@ -909,6 +969,29 @@ async def _run_translation_job(
                ollama_model,
                full_prompt,
            )
+        elif _p == "google_cloud":
+            from services.providers.google_cloud_provider import GoogleCloudTranslationProvider
+            gc_key = _cfg(
+                getattr(_admin_cfg.google_cloud, "api_key", None),
+                "GOOGLE_CLOUD_API_KEY",
+            )
+            if gc_key:
+                translation_provider = GoogleCloudTranslationProvider(
+                    api_key=gc_key,
+                    timeout=int(os.getenv("GOOGLE_CLOUD_TIMEOUT", "30")),
+                    max_retries=int(os.getenv("GOOGLE_CLOUD_MAX_RETRIES", "3")),
+                    retry_delay=float(os.getenv("GOOGLE_CLOUD_RETRY_DELAY", "1.0")),
+                )
+                logger.info(
+                    "google_cloud_provider_selected",
+                    job_id=job_id,
+                )
+            else:
+                logger.warning(
+                    "google_cloud_key_missing_fallback_to_google",
+                    job_id=job_id,
+                )
+                # translation_provider reste None → legacy Google gratuit

        tracker.update(20, "Preparing translation")

@@ -950,12 +1033,12 @@ async def _run_translation_job(
        # Run synchronous translators in a thread pool to avoid blocking the event loop.
        # Without this, status polling requests from the frontend would time out during
        # translation, causing the "Connection lost" error and frozen progress bar.
-        # Always call set_provider (even with None) to reset any previously-set
-        # provider on the singleton translator instances between jobs.
+        # One translator instance per job so concurrent jobs never share mutable
+        # provider state (singleton set_provider was racy under parallel translations).
        if file_extension == ".xlsx":
-            excel_translator.set_provider(translation_provider)
+            job_translator = ExcelTranslator(provider=translation_provider)
            await asyncio.to_thread(
-                excel_translator.translate_file,
+                job_translator.translate_file,
                input_path,
                output_path,
                target_lang,
@@ -963,9 +1046,9 @@ async def _run_translation_job(
                progress_callback=progress_callback,
            )
        elif file_extension == ".docx":
-            word_translator.set_provider(translation_provider)
+            job_translator = WordTranslator(provider=translation_provider)
            await asyncio.to_thread(
-                word_translator.translate_file,
+                job_translator.translate_file,
                input_path,
                output_path,
                target_lang,
@@ -973,9 +1056,9 @@ async def _run_translation_job(
                progress_callback=progress_callback,
            )
        elif file_extension == ".pptx":
-            pptx_translator.set_provider(translation_provider)
+            job_translator = PowerPointTranslator(provider=translation_provider)
            await asyncio.to_thread(
-                pptx_translator.translate_file,
+                job_translator.translate_file,
                input_path,
                output_path,
                target_lang,
@@ -987,6 +1070,12 @@ async def _run_translation_job(

        if user_id:
            await tier_quota_service.increment_on_success(user_id)
+            # Persist monthly usage counters in PostgreSQL (docs + pages)
+            pages = await asyncio.to_thread(
+                _estimate_pages, input_path, file_extension
+            )
+            await asyncio.to_thread(record_usage, user_id, pages)
+            logger.info(f"Job {job_id}: usage recorded — {pages} page(s)")

        tracker.set_completed(str(output_path))
        logger.info(f"Job {job_id}: Completed successfully")
@@ -1095,7 +1184,9 @@ async def get_translation_status(
    }
    ```
    """
-    job = _translation_jobs.get(job_id)
+    job = await get_job_status_async(job_id)
+    if not job:
+        job = _translation_jobs.get(job_id)

    if not job:
        return JSONResponse(
@@ -1231,7 +1322,9 @@ async def download_translated_file(
            },
        )

-    job = _translation_jobs.get(job_id)
+    job = await get_job_status_async(job_id)
+    if not job:
+        job = _translation_jobs.get(job_id)

    if not job:
        return JSONResponse(