From 5fd087979b5ada22756af2bd6dd1aaae00298242 Mon Sep 17 00:00:00 2001 From: sepehr Date: Sun, 14 Jun 2026 10:44:46 +0200 Subject: [PATCH] feat: unify multimodels translation providers, remove self-hosting (Ollama/LibreTranslate), and fix local SQLite configuration --- ARCHITECTURE.md | 12 +- DEPLOYMENT_HOMELAB.md | 530 ---------------- GUIDE_UTILISATION.md | 22 +- MARKETING_PLAN.md | 9 +- README.md | 6 +- config.py | 6 +- mcp_server.py | 77 +-- middleware/validation.py | 14 +- models/subscription.py | 3 - routes/admin_routes.py | 77 +-- routes/legacy_routes.py | 42 +- routes/translate_routes.py | 87 +-- services/providers/__init__.py | 3 - services/providers/base.py | 18 + services/providers/config.py | 15 - services/providers/ollama_provider.py | 599 ------------------- tests/test_providers/test_fallback.py | 2 +- tests/test_providers/test_ollama_provider.py | 493 --------------- tests/test_story_3_5_api_versioning.py | 11 - tests/test_translate_endpoint.py | 15 - translators/pdf_translator.py | 58 +- 21 files changed, 157 insertions(+), 1942 deletions(-) delete mode 100644 DEPLOYMENT_HOMELAB.md delete mode 100644 services/providers/ollama_provider.py delete mode 100644 tests/test_providers/test_ollama_provider.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index de1e64f..5edffb2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -41,8 +41,8 @@ │ ┌─────────────┼─────────────┐ ▼ ▼ ▼ - Google DeepL LibreTranslate - Translate (API Key) (Self-hosted) + Google DeepL Cloud LLMs + Translate (API Key) (OpenAI, DeepSeek...) ``` ## Component Breakdown @@ -126,10 +126,10 @@ Output: Translated .pptx with identical design - Better context understanding - Requires paid API key -3. **LibreTranslate** (Self-hosted) - - Open-source alternative - - Full control and privacy - - Requires local installation +3. **Cloud LLMs** (OpenAI, DeepSeek, OpenRouter, Minimax, Zai) + - Custom translation prompts support (glossary, style preservation) + - Contextual understanding and formatting preservation + - Requires respective API keys ### 4. Utility Layer diff --git a/DEPLOYMENT_HOMELAB.md b/DEPLOYMENT_HOMELAB.md deleted file mode 100644 index 51ea1b0..0000000 --- a/DEPLOYMENT_HOMELAB.md +++ /dev/null @@ -1,530 +0,0 @@ -# Guide de Deploiement - Wordly.art (Homelab) -> NPM + IONOS + Docker + Stripe + API Keys + NAS Backup + Monitoring - ---- - -## Architecture - -``` -Internet - | - | DNS IONOS: wordly.art -> IP fixe - | -[Routeur/Box] (port forwarding 80+443 -> machine NPM) - | - +-- Machine 1: Nginx Proxy Manager (NPM) - | - SSL Let's Encrypt auto - | - Reverse proxy - | - +-- Machine 2/3: Docker (192.168.1.151) - | - +-- wordly-backend (FastAPI :8000) - +-- wordly-frontend (Next.js :3000) - +-- wordly-postgres (PostgreSQL :5432) - +-- wordly-redis (Redis :6379) - +-- wordly-prometheus (:9090 interne) - +-- wordly-grafana (:3001) - +-- wordly-node-exporter + cadvisor - | - +-- Backup quotidien -> NAS -``` - ---- - -## Etape 1 : DNS IONOS - -1. Se connecter sur **ionos.fr** > **Domaines & SSL** > **wordly.art** > **DNS** -2. Modifier les enregistrements **A** existants : - -| Type | Nom (hote) | Valeur | Attention | -|------|-----------|--------|-----------| -| **A** | *(vide ou @)* | Ton IP fixe | Pas dans "Sous-domaines", dans DNS principal | -| **A** | `www` | Ton IP fixe | Dans Sous-domaines, taper juste `www` | -| **A** | `monitoring` | Ton IP fixe | Dans Sous-domaines, taper juste `monitoring` | - -> Ton IP fixe : ouvre https://api.ipify.org depuis ton reseau - -Ne **pas** toucher aux lignes Mail (MX, SPF, DKIM, autodiscover). - -Verifier apres 10 min : -```bash -nslookup wordly.art -``` - ---- - -## Etape 2 : Port forwarding + NPM - -### 2.1 Routeur : ouvrir les ports - -Sur ton routeur/box (souvent `192.168.1.1`) : - -| Port externe | Port interne | IP destination | -|-------------|-------------|---------------| -| **80** | 80 | IP de la machine NPM | -| **443** | 443 | IP de la machine NPM | - -### 2.2 NPM : creer les Proxy Hosts - -Interface NPM : **http://IP_NPM:81** - -#### Proxy Host 1 : wordly.art - -**Details :** -- Domain Names : `wordly.art` -- Scheme : `http` -- Forward Hostname/IP : `192.168.1.151` (IP du serveur Wordly) -- Forward Port : `3000` -- Cocher Block Common Exploits + Websockets - -**SSL :** -- Request a new SSL Certificate -- Cocher Force SSL + HTTP/2 + HSTS -- Email : `admin@wordly.art` - -**Advanced** - coller cette config nginx : - -```nginx -location /api/ { - proxy_pass http://192.168.1.151:8000; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_connect_timeout 60s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; - add_header Access-Control-Allow-Origin https://wordly.art always; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" always; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-API-Key" always; - add_header Access-Control-Allow-Credentials "true" always; - if ($request_method = 'OPTIONS') { return 204; } -} - -location /translate { - proxy_pass http://192.168.1.151:8000; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_connect_timeout 60s; - proxy_send_timeout 600s; - proxy_read_timeout 600s; - client_max_body_size 100M; -} - -location /health { - proxy_pass http://192.168.1.151:8000; - proxy_http_version 1.1; - proxy_set_header Connection ""; -} - -location /docs { proxy_pass http://192.168.1.151:8000; proxy_set_header Host $host; } -location /redoc { proxy_pass http://192.168.1.151:8000; proxy_set_header Host $host; } -location /openapi.json { proxy_pass http://192.168.1.151:8000; proxy_set_header Host $host; } -location /admin { proxy_pass http://192.168.1.151:3000; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; } -location /_next/static/ { proxy_pass http://192.168.1.151:3000; add_header Cache-Control "public, max-age=31536000, immutable"; } -``` - -> Si NPM et Wordly sont sur la meme machine Docker, remplace `192.168.1.151:8000` par `wordly-backend:8000` etc. - -#### Proxy Host 2 : www.wordly.art - -Meme config, meme certificat SSL. Dans Advanced, ajoute : -```nginx -return 301 https://wordly.art$request_uri; -``` - -#### Proxy Host 3 : monitoring.wordly.art - -- Forward : `192.168.1.151:3001` -- SSL Let's Encrypt - ---- - -## Etape 3 : Serveur Docker (192.168.1.151) - -### 3.1 Installer Docker - -```bash -curl -fsSL https://get.docker.com | sh -sudo usermod -aG docker $USER -newgrp docker -``` - -### 3.2 Cloner le projet - -```bash -git clone -b production-deployment https://gitea.parsanet.org/sepehr/office_translator.git /opt/wordly -cd /opt/wordly -``` - -### 3.3 Configurer le .env avec le wizard - -```bash -bash scripts/setup-env.sh -``` - -Le wizard demande le domaine, le mot de passe admin, le service de traduction, et genere tous les secrets automatiquement. - -Ou configurer manuellement : -```bash -cp .env.production .env -nano .env -``` - -### 3.4 Lancer l'application - -```bash -docker compose up -d --build -docker compose ps # Verifier que tout est "Up (healthy)" -curl http://localhost:8000/health -``` - ---- - -## Etape 4 : Cles API - Services de traduction - -L'application supporte **7 providers de traduction**. Tu n'es pas oblige de tous les configurer - commence avec 1 ou 2. - -### Gerer les cles - -```bash -cd /opt/wordly -bash scripts/manage-keys.sh -``` - -Menu interactif pour ajouter/modifier/supprimer des cles API. - -### Ou trouver les cles - -| Provider | URL d'inscription | Cle a recuperer | Cout | -|----------|-------------------|-----------------|------| -| **Google** | Aucune inscription | Aucune cle (gratuit via deep_translator) | Gratuit | -| **DeepL** | https://www.deepl.com/pro-api | Clé API DeepL (format: `xxx-xxx-...`) | Gratuit 500k car/mois, puis 5.49EUR/mois | -| **OpenAI** | https://platform.openai.com/api-keys | `sk-...` | ~0.15$/1000 traductions | -| **DeepSeek** | https://platform.deepseek.com/api_keys | `sk-...` | ~0.14$/M tokens (tres bon rapport Q/P) | -| **Minimax** | https://platform.minimaxi.com/ | Cle API | Variable | -| **OpenRouter** | https://openrouter.ai/keys | `sk-or-...` | Multi-modeles, pay-per-use | - -### Activer un provider dans le .env - -Exemple pour DeepSeek : - -```bash -# Dans le .env -DEEPSEEK_ENABLED=true -DEEPSEEK_API_KEY=sk-votre-cle-ici -DEEPSEEK_MODEL=deepseek-chat -DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 -``` - -Puis `docker compose restart backend`. - -### Recommandation pour demarrer - -1. **Google** (active par defaut, gratuit, aucune cle) -2. **DeepSeek** (ajouter quand tu veux meilleure qualite, tres peu cher) - ---- - -## Etape 5 : Stripe - Systeme de paiement - -Stripe gere les abonnements (Starter, Pro, Business). Si tu ne configures pas Stripe, l'app fonctionne en mode gratuit sans limitation d'abonnement. - -### 5.1 Creer un compte Stripe - -1. Aller sur **https://dashboard.stripe.com/register** -2. Creer un compte avec ton email -3. Activer le compte (verifie identite + banque pour recevoir les paiements) - -### 5.2 Recuperer les cles Stripe - -Dans le dashboard Stripe : **Developers** > **API keys** - -| Cle | Nom dans .env | Description | -|-----|---------------|-------------| -| `sk_test_...` ou `sk_live_...` | `STRIPE_SECRET_KEY` | Cle secrete (attention: NE PAS utiliser la cle publique `pk_...`) | -| `whsec_...` | `STRIPE_WEBHOOK_SECRET` | Cle du webhook (voir section 5.4) | - -> **Test vs Live** : commence en mode test (`sk_test_...`), passe en live quand tout fonctionne. - -### 5.3 Creer les produits et forfaits - -Dans Stripe : **Products** > **Add product** - -Cree 3 produits : - -**Produit 1 : Starter** -- Name : `Starter` -- Price : `9.00 EUR` / Monthly (recurring) -- Une fois cree, clique sur le prix et copie le **Price ID** (`price_...`) - -**Produit 2 : Pro** -- Name : `Pro` -- Price : `19.00 EUR` / Monthly - -**Produit 3 : Business** -- Name : `Business` -- Price : `49.00 EUR` / Monthly - -Copie les 3 Price IDs. - -### 5.4 Configurer le Webhook Stripe - -Le webhook permet a Stripe de notifier ton app quand un paiement reussit, un abonnement est annule, etc. - -Dans Stripe : **Developers** > **Webhooks** > **Add endpoint** - -- **Endpoint URL** : `https://wordly.art/api/v1/stripe/webhook` -- **Events a ecouter** (cliquer "Select events") : - - `checkout.session.completed` - - `customer.subscription.updated` - - `customer.subscription.deleted` - - `invoice.payment_succeeded` - - `invoice.payment_failed` - -Une fois le webhook cree, clique dessus et copie le **Signing secret** (`whsec_...`). - -### 5.5 Ajouter les cles Stripe dans le .env - -```bash -cd /opt/wordly -bash scripts/manage-keys.sh -# Choisir option 4 (Stripe) -``` - -Ou manuellement dans `.env` : - -```bash -STRIPE_SECRET_KEY=sk_live_votre_cle_secrete -STRIPE_WEBHOOK_SECRET=whsec_votre_signing_secret -STRIPE_STARTER_PRICE_ID=price_votre_price_id_starter -STRIPE_PRO_PRICE_ID=price_votre_price_id_pro -STRIPE_BUSINESS_PRICE_ID=price_votre_price_id_business -``` - -Puis `docker compose restart backend`. - -### 5.6 Tester Stripe en mode test - -1. Utilise `sk_test_...` comme STRIPE_SECRET_KEY -2. Cree des produits en mode test (meme process) -3. Pour tester un paiement, Stripe fournit des cartes test : - - **Succes** : `4242 4242 4242 4242` - - **Echec** : `4000 0000 0000 0002` - - Exp : n'importe quelle date future - - CVC : n'importe quel nombre -4. Verifie dans Stripe Dashboard > Payments que les paiements apparaissent - -### 5.7 Passer en production - -Quand tout fonctionne en test : - -1. Dans Stripe Dashboard, clique **"Activate your account"** en haut a gauche -2. Remplie les infos business (identite, banque) -3. Change dans le .env : `sk_test_...` -> `sk_live_...` -4. Refais les webhooks et produits en mode live -5. `docker compose restart backend` - -### 5.8 Checklist Stripe - -- [ ] Compte Stripe cree et verifie -- [ ] 3 produits crees (Starter 9EUR, Pro 19EUR, Business 49EUR) -- [ ] 3 Price IDs copies -- [ ] STRIPE_SECRET_KEY configure -- [ ] Webhook configure avec les 5 events -- [ ] STRIPE_WEBHOOK_SECRET configure -- [ ] 3 STRIPE_*_PRICE_ID configures -- [ ] Backend redemarre -- [ ] Test avec carte 4242 reussi - ---- - -## Etape 6 : Backup automatique vers NAS - -### 6.1 Monter le NAS - -```bash -sudo apt install cifs-utils -sudo mkdir -p /mnt/nas-backups/wordly - -sudo tee /etc/nas-credentials <> /var/log/wordly-backup.log 2>&1 -``` - ---- - -## Etape 7 : Monitoring (Prometheus + Grafana) - -### 7.1 Lancer - -```bash -cd /opt/wordly -docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d -``` - -### 7.2 Acceder - -- **URL** : `https://monitoring.wordly.art` -- **Login** : `admin` / mot de passe defini dans le .env - -### 7.3 Dashboards - -| Dashboard | Contenu | -|-----------|---------| -| Wordly - Application | Traductions, latence, providers, taux d'erreur | -| Wordly - Infrastructure | CPU, RAM, disque, reseau, status containers | - -Alertes pre-configures : backend down, erreur > 10%, RAM > 90%, disque < 15%. - ---- - -## Etape 8 : Gitea Actions - Deploiement automatique - -### 8.1 Installer le runner sur 192.168.1.151 - -```bash -mkdir -p /opt/gitea-runner && cd /opt/gitea-runner -wget https://gitea.com/gitea/act_runner/releases/latest/download/act_runner-linux-amd64 -chmod +x act_runner-linux-amd64 && mv act_runner-linux-amd64 act_runner -./act_runner generate-config > config.yaml -``` - -### 8.2 Enregistrer le runner - -1. Gitea > office_translator > Settings > Actions > Runners > Create new Runner -2. Copier le token - -```bash -cd /opt/gitea-runner -./act_runner register \ - --instance https://gitea.parsanet.org \ - --token LE_TOKEN \ - --name homelab-runner \ - --labels self-hosted -``` - -### 8.3 Service systemd - -```bash -sudo tee /etc/systemd/system/gitea-runner.service < option 5 | -| Espace disque | `df -h && docker system df` | -| SSL | NPM gere le renouvellement auto | - ---- - -## Checklist complete - -### DNS IONOS -- [ ] A record `@` -> IP fixe -- [ ] A record `www` -> IP fixe -- [ ] A record `monitoring` -> IP fixe -- [ ] Propagation verifiee - -### Routeur + NPM -- [ ] Ports 80+443 ouverts vers machine NPM -- [ ] Proxy Host wordly.art avec custom config -- [ ] SSL Let's Encrypt obtenu -- [ ] Proxy Host monitoring.wordly.art - -### Serveur Docker -- [ ] Docker installe -- [ ] Code clone dans /opt/wordly -- [ ] `bash scripts/setup-env.sh` executed -- [ ] `docker compose up -d --build` reussi -- [ ] Tous les containers healthy - -### Cles API (minimum 1 provider) -- [ ] Provider de traduction configure et teste -- [ ] `bash scripts/manage-keys.sh` pour verifier - -### Stripe (optionnel) -- [ ] Compte Stripe cree et active -- [ ] 3 produits + Price IDs crees -- [ ] Webhook configure (5 events) -- [ ] Cles Stripe dans le .env -- [ ] Test avec carte 4242 reussi - -### Backup NAS -- [ ] NAS monte sur /mnt/nas-backups/wordly -- [ ] Backup manuel OK -- [ ] Cron programme a 3h - -### Monitoring -- [ ] `docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d` -- [ ] Grafana accessible sur monitoring.wordly.art - -### CI/CD -- [ ] Gitea runner installe et enregistre -- [ ] Service systemd active - -### Verification finale -- [ ] https://wordly.art accessible depuis Internet -- [ ] HTTPS OK (cadenas vert) -- [ ] Inscription utilisateur OK -- [ ] Upload + traduction OK -- [ ] Page admin /admin OK -- [ ] Monitoring OK diff --git a/GUIDE_UTILISATION.md b/GUIDE_UTILISATION.md index ece700d..0472bed 100644 --- a/GUIDE_UTILISATION.md +++ b/GUIDE_UTILISATION.md @@ -22,7 +22,7 @@ | Google Translate | Classique | Non (via deep_translator) | | DeepL | Classique | Oui (`DEEPL_API_KEY`) | | OpenAI (GPT-4o-mini) | LLM | Oui (`OPENAI_API_KEY`) | -| Ollama (Llama3) | LLM local | Non (local) | +| DeepSeek / Minimax | LLM | Oui (`DEEPSEEK_API_KEY` / `MINIMAX_API_KEY`) | | OpenRouter | LLM | Oui (`OPENROUTER_API_KEY`) | Les fournisseurs sont utilisés via une **chaîne de fallback** configurable (si un échoue, le suivant prend le relais). @@ -165,21 +165,7 @@ Le frontend est accessible sur http://localhost:3000 ## Services optionnels en production -### Ollama (LLM local) -Pour utiliser un modèle LLM local (Ollama), ajouter le profil `with-ollama` : - -```bash -docker compose --profile with-ollama up -d -``` - -Configurer dans `.env` : - -```env -TRANSLATION_SERVICE=ollama -OLLAMA_BASE_URL=http://ollama:11434 -OLLAMA_MODEL=llama3 -``` ### Monitoring (Prometheus + Grafana) @@ -210,7 +196,7 @@ docker compose --profile with-monitoring up -d 3. **Choisir la langue source** et la **langue cible** 4. **Sélectionner le mode de traduction** : - **Classique** : Google Translate / DeepL (rapide, < 15 sec) - - **LLM** : Ollama / OpenAI (1-3 min, meilleure qualité contextuelle) + - **LLM** : OpenAI / DeepSeek / Minimax (1-3 min, meilleure qualité contextuelle) 5. Cliquer sur **Traduire** 6. Suivre la **barre de progression** en temps réel 7. **Télécharger** le fichier traduit une fois terminé @@ -295,7 +281,7 @@ Générer des clés API pour accéder au service programmatiquement : - Vérifier que le fournisseur de traduction est configuré (`TRANSLATION_SERVICE` dans `.env`) - Pour Google Translate : aucune clé nécessaire - Pour DeepL/OpenAI : vérifier que la clé API est valide -- Pour Ollama : vérifier que le service est lancé et que le modèle est téléchargé +- Pour DeepSeek/Minimax : vérifier que la clé API correspondante est configurée ### Logs @@ -338,7 +324,7 @@ office_translator/ ├── alembic/ # Migrations de base de données ├── routes/ # Endpoints API ├── services/ # Logique métier -│ ├── providers/ # Fournisseurs de traduction (Google, DeepL, OpenAI, Ollama) +│ ├── providers/ # Fournisseurs de traduction (Google, DeepL, OpenAI, DeepSeek, etc.) │ ├── translation_service.py │ └── ... ├── translators/ # Traducteurs de documents (Excel, Word, PowerPoint) diff --git a/MARKETING_PLAN.md b/MARKETING_PLAN.md index 63be3cd..7f218f9 100644 --- a/MARKETING_PLAN.md +++ b/MARKETING_PLAN.md @@ -11,9 +11,8 @@ ### Ce qui nous différencie - **Préservation du format** : Contrairement à Google Translate ou DeepL qui détruisent les layouts complexes, notre moteur maintient la structure exacte -- **Multi-providers** : L'utilisateur choisit son moteur (Google, DeepL, OpenAI, Ollama/local) selon son budget et ses besoins +- **Multi-providers** : L'utilisateur choisit son moteur (Google, DeepL, OpenAI, DeepSeek, OpenRouter, Minimax, Zai) selon son budget et ses besoins - **Glossaires techniques** : Terminologie personnalisée (HVAC, IT, Juridique, Médical) -- **Self-hostable** : Peut être déployé en privé pour les entreprises soucieuses de confidentialité ### Marché cible | Segment | Taille estimée | Priorité | @@ -35,7 +34,7 @@ | 1 | **Page d'accueil / Hero** | Landing page, réseaux sociaux | Montrer l'interface épurée avec le drop-zone de fichier | | 2 | **Upload en cours** | Démonstration du workflow | Fichier Excel chargé avec sélection langue source/cible | | 3 | **Résultat côte à côte** | Preuve de qualité | Document original vs traduit, montrer que le format est intact | -| 4 | **Sélection du provider** | Fonctionnalité clé | Dropdown Google/DeepL/OpenAI/Ollama avec prix affichés | +| 4 | **Sélection du provider** | Fonctionnalité clé | Dropdown Google/DeepL/OpenAI/DeepSeek avec prix affichés | | 5 | **Glossaire technique** | Différenciation | Interface de gestion des glossaires personnalisés | | 6 | **Dashboard admin** | Crédibilité entreprise | Vue monitoring avec statistiques d'utilisation | | 7 | **Page pricing/forfaits** | Conversion | Les 3 tiers (Starter/Pro/Business) clairement affichés | @@ -81,7 +80,7 @@ |-------|--------|----------| | **Landing page** | Mettre en place avec captures d'écran + vidéo + formulaire email | CRITIQUE | | **Product Hunt** | Préparer le launch (assets, description, maker comment) | HAUTE | -| **Reddit** | Posts dans r/SideProject, r/selfhosted, r/translator | HAUTE | +| **Reddit** | Posts dans r/SideProject, r/saas, r/translator | HAUTE | | **Hacker News** | Préparer un "Show HN" technique | HAUTE | | **Twitter/X** | Thread de lancement avec démo GIF | MOYENNE | | **LinkedIn** | Post professionnel ciblant PME et agences | MOYENNE | @@ -192,7 +191,7 @@ | **Google Translate (docs)** | Gratuit, connu | Détruit les formats complexes | Préservation du format | | **DeepL (docs)** | Qualité de traduction | Cher, formatage limité | Multi-provider + prix | | **DocTranslator** | Simple | Qualité inégale, publicité | Interface pro + glossaires | -| **Smartcat** | Complet | Complexe, cher | Simplicité + self-hostable | +| **Smartcat** | Complet | Complexe, cher | Simplicité + multi-providers | | **Transifex** | Enterprise | Trop cher pour PME | Prix accessible | --- diff --git a/README.md b/README.md index ceb5a60..cbc370f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Ce fichier sert de **portail central** pour accéder à toutes les documentation Pour faciliter la navigation, utilisez les liens ci-dessous pour accéder directement aux guides spécialisés : ### 🚀 Démarrage & Utilisation -* 📥 **[Guide de démarrage rapide (QUICKSTART.md)](./QUICKSTART.md)** : Installation locale, configuration d'Ollama et lancement du projet en développement. +* 📥 **[Guide de démarrage rapide (QUICKSTART.md)](./QUICKSTART.md)** : Installation locale et lancement du projet en développement. * 📖 **[Guide d'utilisation de l'API (GUIDE_UTILISATION.md)](./GUIDE_UTILISATION.md)** : Exemples de requêtes de traduction, gestion du glossaire et des presets techniques (CVC, IT, Légal, etc.). * 💡 **[Fiche de référence rapide (QUICK_REFERENCE.md)](./QUICK_REFERENCE.md)** : Commandes utiles, ports réseaux, identifiants par défaut et astuces rapides. @@ -20,7 +20,6 @@ Pour faciliter la navigation, utilisez les liens ci-dessous pour accéder direct ### 🌐 Déploiement en Production * 🏢 **[Guide de Déploiement Général (DEPLOYMENT_GUIDE.md)](./DEPLOYMENT_GUIDE.md)** : Guide standard pour le déploiement de production sous Docker avec reverse-proxy Nginx et SSL. -* 🏠 **[Guide de Déploiement Homelab (DEPLOYMENT_HOMELAB.md)](./DEPLOYMENT_HOMELAB.md)** : Guide pas-à-pas configuré spécifiquement pour le réseau homelab (NPM, Stripe, DNS IONOS, Gitea CI/CD, sauvegarde NAS). * ☁️ **[Déploiement sur IONOS (DEPLOY_IONOS.md)](./DEPLOY_IONOS.md)** : Instructions spécifiques pour déployer l'infrastructure sur un serveur virtuel IONOS. ### 🛡️ Sauvegarde, Résilience & Secours (Disaster Recovery) @@ -34,10 +33,9 @@ Pour faciliter la navigation, utilisez les liens ci-dessous pour accéder direct ### 🔄 Multi-fournisseurs de Traduction L'application supporte 7 moteurs de traduction, activables à la volée : * **Google Translate** (Gratuit, rapide, par défaut) -* **Ollama** (LLM local pour une confidentialité totale, ex : `gemma3`, `llama3.2`) * **DeepL API** (Haute qualité pour l'entreprise) * **OpenAI** (Modèles GPT-4o, support de la vision) -* **DeepSeek, OpenRouter, LibreTranslate, WebLLM** +* **DeepSeek, OpenRouter, Minimax, x.ai (Zai)** (Modèles de pointe pour traductions complexes) ### 📁 Traduction Intelligente par Fichier * **Excel (.xlsx)** : Conserve la fusion des cellules, les formules, les polices de caractères, les styles de bordures et traduit également le texte contenu dans les images (via modèles vision). diff --git a/config.py b/config.py index c7bd948..3291a65 100644 --- a/config.py +++ b/config.py @@ -15,10 +15,6 @@ class Config: TRANSLATION_SERVICE = os.getenv("TRANSLATION_SERVICE", "google") DEEPL_API_KEY = os.getenv("DEEPL_API_KEY", "") - # Ollama Configuration - OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") - OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3") - OLLAMA_VISION_MODEL = os.getenv("OLLAMA_VISION_MODEL", "llava") # ============== File Upload Configuration ============== MAX_FILE_SIZE_MB = int(os.getenv("MAX_FILE_SIZE_MB", "50")) @@ -153,5 +149,5 @@ config = Config() # So that database/connection.py and alembic see DATABASE_URL when only POSTGRES_* is set (AC #1) _effective_db_url = Config._get_database_url() -if _effective_db_url and not os.environ.get("DATABASE_URL", "").strip(): +if Config.ENV == "production" and _effective_db_url and not os.environ.get("DATABASE_URL", "").strip(): os.environ["DATABASE_URL"] = _effective_db_url diff --git a/mcp_server.py b/mcp_server.py index 26c3680..fd1befc 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -43,13 +43,9 @@ class MCPServer: }, "provider": { "type": "string", - "enum": ["google", "ollama", "deepl", "libre"], + "enum": ["google", "google_cloud", "deepl", "openai", "openrouter", "deepseek", "minimax", "zai"], "description": "Translation provider (default: google)" }, - "ollama_model": { - "type": "string", - "description": "Ollama model to use (e.g., 'llama3.2', 'gemma3:12b')" - }, "translate_images": { "type": "boolean", "description": "Extract and translate text from images using vision model" @@ -66,19 +62,7 @@ class MCPServer: "required": ["file_path", "target_language"] } }, - { - "name": "list_ollama_models", - "description": "List available Ollama models for translation", - "inputSchema": { - "type": "object", - "properties": { - "base_url": { - "type": "string", - "description": "Ollama server URL (default: http://localhost:11434)" - } - } - } - }, + { "name": "get_supported_languages", "description": "Get list of supported language codes for translation", @@ -95,16 +79,8 @@ class MCPServer: "properties": { "provider": { "type": "string", - "enum": ["google", "ollama", "deepl", "libre"], + "enum": ["google", "google_cloud", "deepl", "openai", "openrouter", "deepseek", "minimax", "zai"], "description": "Default translation provider" - }, - "ollama_url": { - "type": "string", - "description": "Ollama server URL" - }, - "ollama_model": { - "type": "string", - "description": "Default Ollama model" } } } @@ -124,8 +100,7 @@ class MCPServer: try: if name == "translate_document": return await self.translate_document(arguments) - elif name == "list_ollama_models": - return await self.list_ollama_models(arguments) + elif name == "get_supported_languages": return await self.get_supported_languages() elif name == "configure_translation": @@ -153,8 +128,7 @@ class MCPServer: 'translate_images': str(args.get('translate_images', False)).lower(), } - if args.get('ollama_model'): - data['ollama_model'] = args['ollama_model'] + if args.get('system_prompt'): data['system_prompt'] = args['system_prompt'] @@ -194,29 +168,7 @@ class MCPServer: except requests.exceptions.Timeout: return {"error": "Translation request timed out"} - async def list_ollama_models(self, args: dict) -> dict: - """List available Ollama models""" - base_url = args.get('base_url', 'http://localhost:11434') - - try: - response = requests.get( - f"{self.api_base}/ollama/models", - params={'base_url': base_url}, - timeout=10 - ) - - if response.status_code == 200: - data = response.json() - return { - "models": data.get('models', []), - "count": data.get('count', 0), - "ollama_url": base_url - } - else: - return {"error": "Failed to list models", "models": []} - - except requests.exceptions.ConnectionError: - return {"error": "Cannot connect to API server", "models": []} + async def get_supported_languages(self) -> dict: """Get supported language codes""" @@ -249,22 +201,7 @@ class MCPServer: """Configure translation settings""" config = {} - if args.get('ollama_url') and args.get('ollama_model'): - try: - response = requests.post( - f"{self.api_base}/ollama/configure", - data={ - 'base_url': args['ollama_url'], - 'model': args['ollama_model'] - }, - timeout=10 - ) - - if response.status_code == 200: - config['ollama'] = response.json() - - except Exception as e: - config['ollama_error'] = str(e) + config['provider'] = args.get('provider', 'google') diff --git a/middleware/validation.py b/middleware/validation.py index 846aa15..13af00d 100644 --- a/middleware/validation.py +++ b/middleware/validation.py @@ -513,12 +513,14 @@ class ProviderValidator: SUPPORTED_PROVIDERS = { "google", - "ollama", + "google_cloud", "deepl", - "libre", "openai", - "webllm", "openrouter", + "openrouter_premium", + "deepseek", + "minimax", + "zai", "classic", "llm", } @@ -558,11 +560,7 @@ class ProviderValidator: code="missing_openai_key", ) - elif normalized == "ollama": - # Ollama doesn't require API key but may need model - model = kwargs.get("ollama_model", "") - if not model: - logger.warning("No Ollama model specified, will use default") + return {"provider": normalized, "validated": True} diff --git a/models/subscription.py b/models/subscription.py index b9d83da..80a1525 100644 --- a/models/subscription.py +++ b/models/subscription.py @@ -227,9 +227,6 @@ class User(BaseModel): default_target_lang: str = "en" default_provider: str = "google" - # Ollama self-hosted config - ollama_endpoint: Optional[str] = None - ollama_model: Optional[str] = None class UserCreate(BaseModel): diff --git a/routes/admin_routes.py b/routes/admin_routes.py index e9f1fb3..fe71630 100644 --- a/routes/admin_routes.py +++ b/routes/admin_routes.py @@ -646,10 +646,8 @@ async def update_default_provider( "google", "deepl", "openai", - "ollama", "openrouter", "zai", - "libre", "classic", "llm", ] @@ -848,7 +846,7 @@ class SettingsConfig(BaseModel): google_cloud: ProviderSettings = ProviderSettings() # Cloud Translation API v2 (clé API) deepl: ProviderSettings = ProviderSettings() openai: ProviderSettings = ProviderSettings() - ollama: ProviderSettings = ProviderSettings() # dev-only in UI + openrouter: ProviderSettings = ProviderSettings() # "Traduction IA Essentielle" openrouter_premium: ProviderSettings = ProviderSettings() # "Traduction IA Premium" deepseek: ProviderSettings = ProviderSettings() @@ -924,7 +922,7 @@ async def get_settings(admin_id: str = Depends(require_admin)): payload["deepseek"] = _merge_env(settings.deepseek, key_env="DEEPSEEK_API_KEY", model_env="DEEPSEEK_MODEL", default_model="deepseek-chat") payload["minimax"] = _merge_env(settings.minimax, key_env="MINIMAX_API_KEY", model_env="MINIMAX_MODEL", default_model="abab6.5s-chat") payload["zai"] = _merge_env(settings.zai, key_env="ZAI_API_KEY", model_env="ZAI_MODEL", url_env="ZAI_BASE_URL", default_model="grok-2-1212", default_url="https://api.x.ai/v1") - payload["ollama"] = _merge_env(settings.ollama, url_env="OLLAMA_BASE_URL", model_env="OLLAMA_MODEL", default_url="http://localhost:11434", default_model="llama3") + payload["google_cloud"] = _merge_env(settings.google_cloud, key_env="GOOGLE_CLOUD_API_KEY") # SMTP: merge from env vars, but never expose password @@ -958,7 +956,7 @@ async def get_settings(admin_id: str = Depends(require_admin)): "deepseek": bool(os.getenv("DEEPSEEK_API_KEY", "").strip()), "minimax": bool(os.getenv("MINIMAX_API_KEY", "").strip()), "zai": bool(os.getenv("ZAI_API_KEY", "").strip()), - "ollama": bool(os.getenv("OLLAMA_BASE_URL", "").strip()), + "google_cloud": bool(os.getenv("GOOGLE_CLOUD_API_KEY", "").strip()), "smtp": bool(os.getenv("SMTP_HOST", "").strip()), } @@ -1116,22 +1114,6 @@ async def test_provider( content={"available": True, "models_count": len(models)}, ) - elif provider == "ollama": - import requests as _requests - - base_url = (provider_config.base_url or "").strip() or os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") - resp = _requests.get(f"{base_url}/api/tags", timeout=5) - if resp.ok: - return JSONResponse( - status_code=200, - content={ - "available": True, - "models": resp.json().get("models", []), - }, - ) - return JSONResponse( - status_code=500, content={"available": False, "error": str(resp.text)} - ) elif provider in ("openrouter", "openrouter_premium"): api_key = _key(provider_config.api_key, "OPENROUTER_API_KEY") @@ -1325,59 +1307,6 @@ async def test_send_email( ) -@router.get("/providers/ollama/models") -async def list_ollama_models( - base_url: Optional[str] = Query(None), - admin_id: str = Depends(require_admin), -): - """List available models from Ollama server. Accepts optional base_url query param - so the frontend can pass the URL currently being edited (before save).""" - import requests - from config import config as app_config - - settings = load_settings() - resolved = ( - base_url - or settings.ollama.base_url - or app_config.OLLAMA_BASE_URL - or "http://localhost:11434" - ) - - try: - response = requests.get(f"{resolved}/api/tags", timeout=5) - if response.ok: - data = response.json() - models = [] - for model in data.get("models", []): - models.append( - { - "name": model.get("name", ""), - "size": model.get("size", 0), - "modified_at": model.get("modified_at", ""), - } - ) - return JSONResponse(status_code=200, content={"data": models, "meta": {}}) - return JSONResponse( - status_code=500, - content={ - "error": "OLLAMA_UNAVAILABLE", - "message": f"Ollama returned: {response.text}", - }, - ) - except requests.exceptions.ConnectionError: - return JSONResponse( - status_code=503, - content={ - "error": "OLLAMA_CONNECTION_ERROR", - "message": f"Cannot connect to Ollama at {resolved}", - }, - ) - except Exception as e: - logger.error(f"List Ollama models failed: {e}") - return JSONResponse( - status_code=500, - content={"error": "INTERNAL_ERROR", "message": str(e)}, - ) @router.get("/providers/openai/models") diff --git a/routes/legacy_routes.py b/routes/legacy_routes.py index 902e446..e98b8a5 100644 --- a/routes/legacy_routes.py +++ b/routes/legacy_routes.py @@ -66,8 +66,7 @@ async def get_available_providers(): from routes.admin_routes import load_settings settings = load_settings() - is_dev = os.getenv("APP_ENV", "production").lower() == "development" or \ - os.getenv("SHOW_OLLAMA", "false").lower() == "true" + is_dev = os.getenv("APP_ENV", "production").lower() == "development" def _key_ready(key_var: str) -> bool: return bool(os.getenv(key_var, "").strip()) @@ -208,22 +207,7 @@ async def get_available_providers(): "model": model, }) - # Ollama — dev only - if is_dev and _is_enabled("ollama", url_var="OLLAMA_BASE_URL"): - oll_cfg = getattr(settings, "ollama", None) - model = _resolve_model( - oll_cfg.model if oll_cfg else None, - "OLLAMA_MODEL", - "llama3", - ) - available.append({ - "id": "ollama", - "label": "Ollama (Local)", - "description": "Modèle LLM local — développement uniquement", - "mode": "llm", - "tier": "dev", - "model": model, - }) + return JSONResponse( status_code=200, @@ -640,29 +624,7 @@ async def reconstruct_document( ) -@router.get("/ollama/models") -async def list_ollama_models(base_url: Optional[str] = None): - """List available Ollama models""" - from services.translation_service import OllamaTranslationProvider - url = base_url or config.OLLAMA_BASE_URL - models = OllamaTranslationProvider.list_models(url) - - return {"ollama_url": url, "models": models, "count": len(models)} - - -@router.post("/ollama/configure") -async def configure_ollama(base_url: str = Form(...), model: str = Form(...)): - """Configure Ollama settings""" - config.OLLAMA_BASE_URL = base_url - config.OLLAMA_MODEL = model - - return { - "status": "success", - "message": "Ollama configuration updated", - "ollama_url": base_url, - "model": model, - } @router.get("/metrics") diff --git a/routes/translate_routes.py b/routes/translate_routes.py index 8219749..662e0fa 100644 --- a/routes/translate_routes.py +++ b/routes/translate_routes.py @@ -1030,6 +1030,13 @@ async def _run_translation_job( source_lang=glossary_source_lang, target_lang=target_lang, ) + from services.providers.google_provider import GoogleTranslationProvider + from services.providers.google_cloud_provider import GoogleCloudTranslationProvider + from services.providers.deepl_provider import DeepLTranslationProvider + from services.providers.openai_provider import OpenAITranslationProvider + from services.providers.deepseek_provider import DeepSeekTranslationProvider + from services.providers.minimax_provider import MinimaxTranslationProvider + translation_provider = None _p = provider.lower() @@ -1037,24 +1044,34 @@ async def _run_translation_job( # If the Cloud API key is invalid or the API is not enabled, fall back # to the free legacy Google Translate (deep_translator) instead of failing. if _p == "google": - # the user might have set GOOGLE_API_KEY instead of GOOGLE_CLOUD_API_KEY gc_key = _cfg( getattr(_admin_cfg.google_cloud, "api_key", None), "GOOGLE_CLOUD_API_KEY", ) or os.getenv("GOOGLE_API_KEY", "").strip() if gc_key and _google_cloud_key_valid(gc_key, job_id): - from services.providers.google_cloud_provider import LegacyGoogleCloudAdapter - translation_provider = LegacyGoogleCloudAdapter(gc_key) + translation_provider = GoogleCloudTranslationProvider( + api_key=gc_key, + timeout=int(os.getenv("GOOGLE_CLOUD_TIMEOUT", "30")), + max_retries=int(os.getenv("GOOGLE_CLOUD_MAX_RETRIES", "3")), + retry_delay=float(os.getenv("GOOGLE_CLOUD_RETRY_DELAY", "1.0")), + ) logger.info("google_provider_using_cloud_api", extra={"job_id": job_id}) else: - from services.translation_service import GoogleTranslationProvider - translation_provider = GoogleTranslationProvider() + translation_provider = GoogleTranslationProvider( + use_cache=True, + timeout=int(os.getenv("GOOGLE_TRANSLATE_TIMEOUT", "30")), + max_retries=int(os.getenv("GOOGLE_TRANSLATE_MAX_RETRIES", "3")), + retry_delay=float(os.getenv("GOOGLE_TRANSLATE_RETRY_DELAY", "1.0")), + ) logger.info("google_provider_using_legacy", extra={"job_id": job_id}) elif _p in ("openrouter", "llm") and api_key: - translation_provider = OpenRouterTranslationProvider( - api_key, model, full_prompt + translation_provider = OpenAITranslationProvider( + api_key=api_key, + model=model, + base_url="https://openrouter.ai/api/v1", + timeout=int(os.getenv("OPENROUTER_TIMEOUT", "60")), ) elif _p == "openrouter_premium": premium_key = _cfg(_admin_cfg.openrouter_premium.api_key, "OPENROUTER_API_KEY") @@ -1062,71 +1079,58 @@ async def _run_translation_job( if not premium_key: premium_key = api_key # fall back to main openrouter key if premium_key: - translation_provider = OpenRouterTranslationProvider( - premium_key, premium_model, full_prompt + translation_provider = OpenAITranslationProvider( + api_key=premium_key, + model=premium_model, + base_url="https://openrouter.ai/api/v1", + timeout=int(os.getenv("OPENROUTER_TIMEOUT", "60")), ) elif _p == "openai": - from services.translation_service import OpenAITranslationProvider openai_key = _cfg(_admin_cfg.openai.api_key, "OPENAI_API_KEY") openai_model = _cfg(_admin_cfg.openai.model, "OPENAI_MODEL", "gpt-4o-mini") if openai_key: translation_provider = OpenAITranslationProvider( api_key=openai_key, model=openai_model, - system_prompt=full_prompt, + timeout=int(os.getenv("OPENAI_TIMEOUT", "60")), ) elif _p == "deepseek": - from services.translation_service import OpenAITranslationProvider as _OAI ds_key = _cfg(getattr(_admin_cfg, "deepseek", None) and _admin_cfg.deepseek.api_key, "DEEPSEEK_API_KEY") ds_model = _cfg(getattr(_admin_cfg, "deepseek", None) and _admin_cfg.deepseek.model, "DEEPSEEK_MODEL", "deepseek-chat") - ds_url = "https://api.deepseek.com/v1" if ds_key: - translation_provider = _OAI( + translation_provider = DeepSeekTranslationProvider( api_key=ds_key, model=ds_model, - base_url=ds_url, - system_prompt=full_prompt, + timeout=int(os.getenv("DEEPSEEK_TIMEOUT", "60")), ) elif _p == "minimax": - from services.translation_service import OpenAITranslationProvider as _OAI mm_key = _cfg(getattr(_admin_cfg, "minimax", None) and _admin_cfg.minimax.api_key, "MINIMAX_API_KEY") - mm_model = _cfg(getattr(_admin_cfg, "minimax", None) and _admin_cfg.minimax.model, "MINIMAX_MODEL", "abab6.5s-chat") - mm_url = "https://api.minimax.chat/v1" + mm_model = _cfg(getattr(_admin_cfg, "minimax", None) and _admin_cfg.minimax.model, "MINIMAX_MODEL", "MiniMax-M1") if mm_key: - translation_provider = _OAI( + translation_provider = MinimaxTranslationProvider( api_key=mm_key, model=mm_model, - base_url=mm_url, - system_prompt=full_prompt, + timeout=int(os.getenv("MINIMAX_TIMEOUT", "60")), ) elif _p == "deepl": deepl_key = _cfg(_admin_cfg.deepl.api_key, "DEEPL_API_KEY") if deepl_key: - from services.translation_service import DeepLTranslationProvider - translation_provider = DeepLTranslationProvider(deepl_key) + translation_provider = DeepLTranslationProvider( + api_key=deepl_key, + timeout=int(os.getenv("DEEPL_TIMEOUT", "30")), + ) elif _p == "zai": - from services.translation_service import OpenAITranslationProvider as _OAI zai_key = _cfg(_admin_cfg.zai.api_key, "ZAI_API_KEY") zai_model = _cfg(_admin_cfg.zai.model, "ZAI_MODEL", "grok-2-1212") zai_url = _cfg(_admin_cfg.zai.base_url, "ZAI_BASE_URL", "https://api.x.ai/v1") if zai_key: - translation_provider = _OAI( + translation_provider = OpenAITranslationProvider( api_key=zai_key, model=zai_model, base_url=zai_url, - system_prompt=full_prompt, + timeout=int(os.getenv("ZAI_TIMEOUT", "60")), ) - elif _p == "ollama": - ollama_url = _cfg(_admin_cfg.ollama.base_url, "OLLAMA_BASE_URL", "http://localhost:11434") - ollama_model = _cfg(_admin_cfg.ollama.model, "OLLAMA_MODEL", "llama3") - translation_provider = OllamaTranslationProvider( - ollama_url, - ollama_model, - ollama_model, - full_prompt, - ) elif _p == "google_cloud": - from services.providers.google_cloud_provider import GoogleCloudTranslationProvider gc_key = _cfg( getattr(_admin_cfg.google_cloud, "api_key", None), "GOOGLE_CLOUD_API_KEY", @@ -1147,7 +1151,6 @@ async def _run_translation_job( "google_cloud_key_missing_fallback_to_google", extra={"job_id": job_id}, ) - # translation_provider reste None → legacy Google gratuit tracker.update(20, "Preparing translation") @@ -1194,6 +1197,8 @@ async def _run_translation_job( if file_extension == ".xlsx": logger.info(f"DEBUG: ExcelTranslator class is {ExcelTranslator} and translate_file is {ExcelTranslator.translate_file}") job_translator = ExcelTranslator(provider=translation_provider) + if hasattr(job_translator, "set_custom_prompt"): + job_translator.set_custom_prompt(full_prompt) await asyncio.to_thread( job_translator.translate_file, input_path, @@ -1205,6 +1210,8 @@ async def _run_translation_job( ) elif file_extension == ".docx": job_translator = WordTranslator(provider=translation_provider) + if hasattr(job_translator, "set_custom_prompt"): + job_translator.set_custom_prompt(full_prompt) await asyncio.to_thread( job_translator.translate_file, input_path, @@ -1216,6 +1223,8 @@ async def _run_translation_job( ) elif file_extension == ".pptx": job_translator = PowerPointTranslator(provider=translation_provider) + if hasattr(job_translator, "set_custom_prompt"): + job_translator.set_custom_prompt(full_prompt) await asyncio.to_thread( job_translator.translate_file, input_path, @@ -1228,6 +1237,8 @@ async def _run_translation_job( elif file_extension == ".pdf": from translators.pdf_translator import PDFTranslator job_translator = PDFTranslator(provider=translation_provider) + if hasattr(job_translator, "set_custom_prompt"): + job_translator.set_custom_prompt(full_prompt) actual_output = await asyncio.to_thread( job_translator.translate_file, input_path, diff --git a/services/providers/__init__.py b/services/providers/__init__.py index d6e9d89..39c6de7 100644 --- a/services/providers/__init__.py +++ b/services/providers/__init__.py @@ -59,10 +59,7 @@ def _auto_register_providers() -> None: register_deepl_provider() - if ProvidersConfig.OLLAMA_ENABLED: - from .ollama_provider import register_ollama_provider - register_ollama_provider() if ProvidersConfig.OPENAI_ENABLED and ProvidersConfig.OPENAI_API_KEY: from .openai_provider import register_openai_provider diff --git a/services/providers/base.py b/services/providers/base.py index 8710213..d89206c 100644 --- a/services/providers/base.py +++ b/services/providers/base.py @@ -73,6 +73,24 @@ class TranslationProvider(ABC): """ return [self.translate_text(req) for req in requests] + def translate( + self, text: str, target_language: str, source_language: str = "auto" + ) -> str: + """ + Compatibility method for the legacy interface. + Translates a single text string synchronously. + """ + req = TranslationRequest( + text=text, + target_language=target_language, + source_language=source_language, + ) + resp = self.translate_text(req) + if resp.error: + raise Exception(f"[{resp.error_code or 'UNKNOWN'}] {resp.error}") + return resp.translated_text + + def health_check(self) -> ProviderHealthStatus: """ Return health status details for the provider. diff --git a/services/providers/config.py b/services/providers/config.py index 0769827..327113e 100644 --- a/services/providers/config.py +++ b/services/providers/config.py @@ -79,15 +79,6 @@ class ProvidersConfig: os.getenv("OPENAI_HEALTH_CHECK_TIMEOUT", "5") ) - # Ollama (local LLM) - default model is config-only, no hardcode in provider - _DEFAULT_OLLAMA_MODEL: str = "llama3" - OLLAMA_ENABLED: bool = os.getenv("OLLAMA_ENABLED", "false").lower() == "true" - OLLAMA_BASE_URL: str = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") - OLLAMA_MODEL: str = os.getenv("OLLAMA_MODEL", _DEFAULT_OLLAMA_MODEL) - OLLAMA_VISION_MODEL: str = os.getenv("OLLAMA_VISION_MODEL", "llava") - OLLAMA_TIMEOUT: int = int(os.getenv("OLLAMA_TIMEOUT", "120")) - OLLAMA_MAX_RETRIES: int = int(os.getenv("OLLAMA_MAX_RETRIES", "2")) - OLLAMA_RETRY_DELAY: float = float(os.getenv("OLLAMA_RETRY_DELAY", "2.0")) # OpenRouter (multi-model API) OPENROUTER_ENABLED: bool = ( @@ -193,12 +184,6 @@ class ProvidersConfig: base_url=cls.OPENAI_BASE_URL or None, model=cls.OPENAI_MODEL, ), - "ollama": ProviderSettings( - enabled=cls.OLLAMA_ENABLED, - api_key=None, - base_url=cls.OLLAMA_BASE_URL, - model=cls.OLLAMA_MODEL, - ), "openrouter": ProviderSettings( enabled=cls.OPENROUTER_ENABLED, api_key=cls.OPENROUTER_API_KEY if cls.OPENROUTER_API_KEY else None, diff --git a/services/providers/ollama_provider.py b/services/providers/ollama_provider.py deleted file mode 100644 index 8056b48..0000000 --- a/services/providers/ollama_provider.py +++ /dev/null @@ -1,599 +0,0 @@ -""" -Ollama Provider - Local LLM translation provider. - -Extends TranslationProvider base class with robust error handling, -retry logic, and health monitoring for local Ollama instances. - -Features: -- Local LLM translation via Ollama REST API -- Custom system prompt support -- Specific error codes for all Ollama API errors -- Retry logic with exponential backoff for transient errors -- Timeout configuration (longer for LLM) -- Health check with caching -- Structlog-compatible logging (no document content in logs) -""" - -import socket -import threading -import time -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from urllib.parse import urljoin - -from core.logging import get_logger - -logger = get_logger(__name__) -_HAS_STRUCTLOG = True - - -def _log_info(event: str, **kwargs): - """Log info with structlog or standard logging compatibility.""" - if _HAS_STRUCTLOG: - logger.info(event, **kwargs) - else: - msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items()) - logger.info(msg) - - -def _log_warning(event: str, **kwargs): - """Log warning with structlog or standard logging compatibility.""" - if _HAS_STRUCTLOG: - logger.warning(event, **kwargs) - else: - msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items()) - logger.warning(msg) - - -def _log_error(event: str, **kwargs): - """Log error with structlog or standard logging compatibility.""" - if _HAS_STRUCTLOG: - logger.error(event, **kwargs) - else: - msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items()) - logger.error(msg) - - -import requests -from requests.exceptions import Timeout, ConnectionError as RequestsConnectionError - -from .base import TranslationProvider -from .schemas import ( - ProviderHealthStatus, - TranslationRequest, - TranslationResponse, -) - -OLLAMA_UNAVAILABLE = "OLLAMA_UNAVAILABLE" -OLLAMA_MODEL_NOT_FOUND = "OLLAMA_MODEL_NOT_FOUND" -OLLAMA_TIMEOUT = "OLLAMA_TIMEOUT" -OLLAMA_GENERATION_ERROR = "OLLAMA_GENERATION_ERROR" -OLLAMA_CONTEXT_TOO_LONG = "OLLAMA_CONTEXT_TOO_LONG" - -_RETRYABLE_ERRORS = {OLLAMA_UNAVAILABLE, OLLAMA_TIMEOUT} - - -class OllamaProviderError(Exception): - """Exception raised for Ollama API errors.""" - - def __init__( - self, code: str, message: str, details: Optional[Dict[str, Any]] = None - ): - self.code = code - self.message = message - self.details = details or {} - super().__init__(message) - - def to_dict(self) -> Dict[str, Any]: - """Convert error to dictionary format.""" - result = { - "error": self.code, - "message": self.message, - } - if self.details: - result["details"] = self.details - return result - - -DEFAULT_TRANSLATION_PROMPT = """You are a professional translator. Translate the following text from {source_lang} to {target_lang}. - -Rules: -- Translate ONLY the text, do not add explanations or notes -- Preserve the original formatting, line breaks, and structure -- Maintain the original tone and style -- For technical terms, use the standard translation in the target language -- If the text contains proper nouns or brand names, keep them unchanged unless there's a well-known translation""" - - -def _build_system_prompt( - source_lang: str, target_lang: str, custom_prompt: Optional[str] = None -) -> str: - """Build system prompt for translation.""" - if custom_prompt: - return custom_prompt - return DEFAULT_TRANSLATION_PROMPT.format( - source_lang=source_lang, target_lang=target_lang - ) - - -def _get_language_name(code: str) -> str: - """Convert language code to full name for better LLM understanding.""" - language_names = { - "en": "English", - "fr": "French", - "es": "Spanish", - "de": "German", - "it": "Italian", - "pt": "Portuguese", - "nl": "Dutch", - "ru": "Russian", - "zh": "Chinese", - "ja": "Japanese", - "ko": "Korean", - "ar": "Arabic", - "hi": "Hindi", - "tr": "Turkish", - "pl": "Polish", - "vi": "Vietnamese", - "th": "Thai", - "id": "Indonesian", - "ms": "Malay", - "uk": "Ukrainian", - "cs": "Czech", - "sv": "Swedish", - "da": "Danish", - "fi": "Finnish", - "no": "Norwegian", - "el": "Greek", - "he": "Hebrew", - "ro": "Romanian", - "hu": "Hungarian", - "bg": "Bulgarian", - "sk": "Slovak", - "hr": "Croatian", - "sl": "Slovenian", - "lt": "Lithuanian", - "lv": "Latvian", - "et": "Estonian", - } - base_code = code.split("-")[0].lower() - return language_names.get(base_code, code) - - -class OllamaTranslationProvider(TranslationProvider): - """ - Ollama LLM implementation for local translation. - - Features: - - Uses Ollama REST API (/api/chat endpoint) - - Custom system prompt support for translation context - - Thread-safe HTTP client - - Robust error handling with specific error codes - - Retry logic with exponential backoff - - Configurable timeout (default 120s for LLM) - - Health check with result caching - """ - - def __init__( - self, - base_url: str = "http://localhost:11434", - model: Optional[str] = None, - timeout: int = 120, - max_retries: int = 2, - retry_delay: float = 2.0, - ): - """ - Initialize Ollama provider. - - Args: - base_url: Ollama API base URL (default: http://localhost:11434) - model: Model name (e.g. llama3, mistral). If None, uses OLLAMA_MODEL from config. - timeout: Request timeout in seconds (default: 120 for LLM) - max_retries: Maximum retry attempts for transient errors (default: 2) - retry_delay: Initial retry delay in seconds (default: 2.0) - """ - if model is None: - from .config import ProvidersConfig - - model = ProvidersConfig.OLLAMA_MODEL - self._base_url = base_url.rstrip("/") - self._model = model - self._provider_name = "ollama" - self.timeout = timeout - self.max_retries = max_retries - self.retry_delay = retry_delay - self._health_cache: Dict[str, Any] = {} - self._health_cache_ttl = 60 - self._health_cache_lock = threading.Lock() - self._available_models: Optional[List[str]] = None - self._models_cache_time: float = 0 - self._models_cache_ttl = 300 - - def _fetch_available_models(self) -> List[str]: - """Fetch list of available (pulled) models from Ollama.""" - current_time = time.time() - - if ( - self._available_models is not None - and current_time - self._models_cache_time < self._models_cache_ttl - ): - return self._available_models - - try: - response = requests.get(f"{self._base_url}/api/tags", timeout=10) - if response.status_code == 200: - data = response.json() - models = [m.get("name", "") for m in data.get("models", [])] - self._available_models = models - self._models_cache_time = current_time - return models - except Exception as e: - _log_warning("ollama_models_fetch_failed", error=str(e)[:100]) - - return [] - - def _check_model_available(self, model: str) -> bool: - """Check if a specific model is available (pulled).""" - models = self._fetch_available_models() - return any(m.startswith(model) or model in m for m in models) - - def _make_api_request(self, text: str, system_prompt: str) -> str: - """ - Make API request to Ollama. - - Raises: - OllamaProviderError: For any API errors with specific codes - """ - if not text or not text.strip(): - return text - - if len(text) > 128000: - raise OllamaProviderError( - code=OLLAMA_CONTEXT_TOO_LONG, - message="Texte trop long pour le modèle (max ~128K caractères).", - details={"text_length": len(text), "max_chars": 128000}, - ) - - if not self._check_model_available(self._model): - raise OllamaProviderError( - code=OLLAMA_MODEL_NOT_FOUND, - message=f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}", - details={"model": self._model, "provider": "ollama"}, - ) - - payload = { - "model": self._model, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": text}, - ], - "stream": False, - "options": {"temperature": 0.3}, - } - - try: - response = requests.post( - f"{self._base_url}/api/chat", - json=payload, - timeout=self.timeout, - ) - - if response.status_code == 404: - raise OllamaProviderError( - code=OLLAMA_MODEL_NOT_FOUND, - message=f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}", - details={"model": self._model, "status_code": 404}, - ) - - if response.status_code != 200: - error_text = response.text[:200] if response.text else "Unknown error" - raise OllamaProviderError( - code=OLLAMA_GENERATION_ERROR, - message=f"Erreur de génération Ollama: {error_text}", - details={"status_code": response.status_code, "model": self._model}, - ) - - data = response.json() - message = data.get("message", {}) - content = message.get("content", "") - - if not content: - raise OllamaProviderError( - code=OLLAMA_GENERATION_ERROR, - message="Erreur de génération Ollama: réponse vide", - details={"model": self._model, "response": str(data)[:200]}, - ) - - return content.strip() - - except Timeout: - raise OllamaProviderError( - code=OLLAMA_TIMEOUT, - message="Délai d'attente Ollama dépassé. Réessayez avec un texte plus court.", - details={"provider": "ollama", "timeout_seconds": self.timeout}, - ) - except RequestsConnectionError: - raise OllamaProviderError( - code=OLLAMA_UNAVAILABLE, - message="Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution.", - details={"provider": "ollama", "base_url": self._base_url}, - ) - except OllamaProviderError: - raise - except Exception as e: - error_str = str(e).lower() - if "connection" in error_str or "refused" in error_str: - raise OllamaProviderError( - code=OLLAMA_UNAVAILABLE, - message="Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution.", - details={"provider": "ollama", "base_url": self._base_url}, - ) - raise OllamaProviderError( - code=OLLAMA_GENERATION_ERROR, - message=f"Erreur de génération Ollama: {str(e)[:100]}", - details={"provider": "ollama", "original_error": str(e)[:100]}, - ) - - def get_name(self) -> str: - """Return provider name.""" - return self._provider_name - - def is_available(self) -> bool: - """ - Check if Ollama is available. - - Uses cached result if available and not expired. - """ - current_time = time.time() - - with self._health_cache_lock: - if "is_available" in self._health_cache: - cached = self._health_cache["is_available"] - if current_time - cached["timestamp"] < self._health_cache_ttl: - return cached["value"] - - try: - response = requests.get(f"{self._base_url}/api/tags", timeout=5) - available = response.status_code == 200 - except Exception as e: - _log_warning("ollama_availability_check_failed", error=str(e)[:100]) - available = False - - with self._health_cache_lock: - self._health_cache["is_available"] = { - "value": available, - "timestamp": current_time, - } - - return available - - def translate_text(self, request: TranslationRequest) -> TranslationResponse: - """ - Translate a single text string using Ollama LLM. - - Supports custom system prompt via request.metadata["custom_prompt"]. - - Args: - request: TranslationRequest with text and language info - - Returns: - TranslationResponse with translated text - """ - text = request.text - target_language = request.target_language - source_language = request.source_language or "auto" - - if not text or not text.strip(): - return TranslationResponse( - translated_text=text, - provider_name=self._provider_name, - from_cache=False, - ) - - source_lang_name = _get_language_name(source_language) - target_lang_name = _get_language_name(target_language) - - custom_prompt = None - if request.metadata: - custom_prompt = request.metadata.get("custom_prompt") - - system_prompt = _build_system_prompt( - source_lang_name, target_lang_name, custom_prompt - ) - - last_error: Optional[OllamaProviderError] = None - retries = 0 - - while retries <= self.max_retries: - try: - start_time = time.time() - result = self._make_api_request(text, system_prompt) - latency = time.time() - start_time - - _log_info( - "ollama_translation_success", - chars=len(text), - source_lang=source_language, - target_lang=target_language, - model=self._model, - latency_ms=round(latency * 1000, 2), - retries=retries, - ) - - return TranslationResponse( - translated_text=result, - provider_name=self._provider_name, - from_cache=False, - source_language=source_language, - ) - - except OllamaProviderError as e: - last_error = e - - if e.code not in _RETRYABLE_ERRORS: - break - - retries += 1 - if retries <= self.max_retries: - delay = self.retry_delay * (2 ** (retries - 1)) - _log_info( - "ollama_translation_retry", - attempt=retries, - delay_s=round(delay, 2), - error_code=e.code, - text_length=len(text), - source_lang=source_language, - target_lang=target_language, - ) - time.sleep(delay) - - except Exception as e: - last_error = OllamaProviderError( - code=OLLAMA_GENERATION_ERROR, - message=f"Erreur de génération Ollama: {str(e)[:100]}", - details={"original_error": str(e)[:100]}, - ) - retries += 1 - if retries <= self.max_retries: - delay = self.retry_delay * (2 ** (retries - 1)) - time.sleep(delay) - - if last_error: - _log_error( - "ollama_translation_failed", - error_code=last_error.code, - text_length=len(text), - source_lang=source_language, - target_lang=target_language, - retries=retries, - ) - return TranslationResponse( - translated_text=text, - provider_name=self._provider_name, - from_cache=False, - error=last_error.message, - error_code=last_error.code, - error_details=last_error.details, - ) - - return TranslationResponse( - translated_text=text, - provider_name=self._provider_name, - from_cache=False, - error="Unknown error", - error_code=OLLAMA_GENERATION_ERROR, - ) - - def translate_batch( - self, requests: List[TranslationRequest] - ) -> List[TranslationResponse]: - """ - Translate multiple texts. - - Args: - requests: List of TranslationRequest objects - - Returns: - List of TranslationResponse objects - """ - if not requests: - return [] - - return [self.translate_text(req) for req in requests] - - def health_check(self) -> ProviderHealthStatus: - """ - Return health status details for the provider. - - Includes cached result for efficiency. - - Returns: - ProviderHealthStatus with availability and latency information - """ - current_time = time.time() - - with self._health_cache_lock: - if "health_check" in self._health_cache: - cached = self._health_cache["health_check"] - if current_time - cached["timestamp"] < self._health_cache_ttl: - return cached["value"] - - start_time = time.time() - last_check_iso = datetime.now(timezone.utc).isoformat() - - try: - models = self._fetch_available_models() - model_available = self._check_model_available(self._model) - available = len(models) > 0 and model_available - latency_ms = (time.time() - start_time) * 1000 - - error_msg = None - if not available and len(models) == 0: - error_msg = "Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution." - elif not model_available: - error_msg = f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}" - - status = ProviderHealthStatus( - name=self._provider_name, - available=available, - latency_ms=round(latency_ms, 2), - error=error_msg, - last_check=last_check_iso, - model=self._model, - model_available=model_available, - ) - except Exception as e: - latency_ms = (time.time() - start_time) * 1000 - status = ProviderHealthStatus( - name=self._provider_name, - available=False, - latency_ms=round(latency_ms, 2), - error=str(e)[:100], - last_check=last_check_iso, - model=self._model, - model_available=None, - ) - - with self._health_cache_lock: - self._health_cache["health_check"] = { - "value": status, - "timestamp": current_time, - } - - return status - - -def register_ollama_provider(): - """ - Register the Ollama provider in the global registry. - - This function should be called during module initialization - to make the provider available through the registry. - """ - from .registry import registry - - provider = get_ollama_provider() - registry.register("ollama", provider) - return provider - - -_provider_instance: Optional[OllamaTranslationProvider] = None -_provider_lock = threading.Lock() - - -def get_ollama_provider() -> OllamaTranslationProvider: - """Get or create the Ollama provider instance (reads config from env).""" - global _provider_instance - if _provider_instance is None: - with _provider_lock: - if _provider_instance is None: - from .config import ProvidersConfig - - _provider_instance = OllamaTranslationProvider( - base_url=ProvidersConfig.OLLAMA_BASE_URL, - model=ProvidersConfig.OLLAMA_MODEL, - timeout=ProvidersConfig.OLLAMA_TIMEOUT, - max_retries=ProvidersConfig.OLLAMA_MAX_RETRIES, - retry_delay=ProvidersConfig.OLLAMA_RETRY_DELAY, - ) - return _provider_instance diff --git a/tests/test_providers/test_fallback.py b/tests/test_providers/test_fallback.py index b18a206..19c40bd 100644 --- a/tests/test_providers/test_fallback.py +++ b/tests/test_providers/test_fallback.py @@ -345,7 +345,7 @@ class TestTranslateWithFallbackByMode: mock_translate.return_value = MagicMock() with patch("services.providers.config.ProvidersConfig") as mock_config: - mock_config.get_fallback_chain.return_value = ["ollama", "openai"] + mock_config.get_fallback_chain.return_value = ["openrouter", "openai"] request = TranslationRequest(text="Hello", target_language="fr") translate_with_fallback_by_mode(request, mode="llm") diff --git a/tests/test_providers/test_ollama_provider.py b/tests/test_providers/test_ollama_provider.py deleted file mode 100644 index 7cf383d..0000000 --- a/tests/test_providers/test_ollama_provider.py +++ /dev/null @@ -1,493 +0,0 @@ -""" -Tests for the OllamaTranslationProvider. -""" - -import pytest -from unittest.mock import patch, MagicMock -from requests.exceptions import Timeout, ConnectionError as RequestsConnectionError - -from services.providers.ollama_provider import ( - OllamaTranslationProvider, - OllamaProviderError, - get_ollama_provider, - register_ollama_provider, - _build_system_prompt, - _get_language_name, - OLLAMA_UNAVAILABLE, - OLLAMA_MODEL_NOT_FOUND, - OLLAMA_TIMEOUT, - OLLAMA_GENERATION_ERROR, - OLLAMA_CONTEXT_TOO_LONG, -) -from services.providers.schemas import TranslationRequest, TranslationResponse - - -class TestOllamaProviderError: - """Tests for OllamaProviderError exception.""" - - def test_error_creation(self): - """Test error creation with all fields.""" - error = OllamaProviderError( - code=OLLAMA_UNAVAILABLE, - message="Ollama unavailable", - details={"provider": "ollama"}, - ) - - assert error.code == OLLAMA_UNAVAILABLE - assert error.message == "Ollama unavailable" - assert error.details == {"provider": "ollama"} - - def test_error_to_dict(self): - """Test error serialization.""" - error = OllamaProviderError( - code=OLLAMA_MODEL_NOT_FOUND, - message="Model not found", - details={"model": "llama3"}, - ) - - result = error.to_dict() - - assert result["error"] == OLLAMA_MODEL_NOT_FOUND - assert result["message"] == "Model not found" - assert result["details"]["model"] == "llama3" - - def test_error_to_dict_no_details(self): - """Test error serialization without details.""" - error = OllamaProviderError( - code=OLLAMA_GENERATION_ERROR, - message="Generation error", - ) - - result = error.to_dict() - - assert result["error"] == OLLAMA_GENERATION_ERROR - assert result["message"] == "Generation error" - assert "details" not in result - - -class TestHelperFunctions: - """Tests for helper functions.""" - - def test_get_language_name_common(self): - """Test language name lookup for common languages.""" - assert _get_language_name("en") == "English" - assert _get_language_name("fr") == "French" - assert _get_language_name("es") == "Spanish" - assert _get_language_name("de") == "German" - assert _get_language_name("zh") == "Chinese" - assert _get_language_name("ja") == "Japanese" - - def test_get_language_name_with_variant(self): - """Test language name lookup with variant codes.""" - assert _get_language_name("en-US") == "English" - assert _get_language_name("pt-BR") == "Portuguese" - - def test_get_language_name_unknown(self): - """Test language name lookup for unknown codes.""" - assert _get_language_name("xx") == "xx" - - def test_build_system_prompt_default(self): - """Test default system prompt generation.""" - prompt = _build_system_prompt("English", "French") - - assert "English" in prompt - assert "French" in prompt - assert "translator" in prompt.lower() - - def test_build_system_prompt_custom(self): - """Test custom system prompt.""" - custom = "Translate this text formally for business context." - prompt = _build_system_prompt("English", "French", custom) - - assert prompt == custom - - -class TestOllamaTranslationProvider: - """Tests for OllamaTranslationProvider.""" - - @pytest.fixture - def provider(self): - """Create an Ollama provider instance.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - timeout=120, - max_retries=0, - ) - - @pytest.fixture - def provider_with_retries(self): - """Create an Ollama provider with retries.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - timeout=120, - max_retries=2, - retry_delay=0.01, - ) - - def test_init(self, provider): - """Test provider initialization.""" - assert provider._base_url == "http://localhost:11434" - assert provider._model == "llama3" - assert provider.timeout == 120 - assert provider._provider_name == "ollama" - - def test_get_name(self, provider): - """Test provider name.""" - assert provider.get_name() == "ollama" - - def test_translate_text_empty(self, provider): - """Test translating empty text.""" - request = TranslationRequest(text="", target_language="fr") - response = provider.translate_text(request) - - assert response.translated_text == "" - assert response.provider_name == "ollama" - assert response.from_cache is False - - def test_translate_text_whitespace(self, provider): - """Test translating whitespace-only text.""" - request = TranslationRequest(text=" ", target_language="fr") - response = provider.translate_text(request) - - assert response.translated_text == " " - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - @patch.object(OllamaTranslationProvider, "_make_api_request") - def test_translate_text_success(self, mock_request, mock_models, provider): - """Test successful translation.""" - mock_models.return_value = ["llama3", "mistral"] - mock_request.return_value = "Bonjour" - - request = TranslationRequest(text="Hello", target_language="fr") - response = provider.translate_text(request) - - assert response.translated_text == "Bonjour" - assert response.provider_name == "ollama" - assert response.from_cache is False - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - @patch.object(OllamaTranslationProvider, "_make_api_request") - def test_translate_text_with_custom_prompt( - self, mock_request, mock_models, provider - ): - """Test translation with custom system prompt.""" - mock_models.return_value = ["llama3"] - mock_request.return_value = "Bonjour (formal)" - - request = TranslationRequest( - text="Hello", - target_language="fr", - metadata={"custom_prompt": "Translate formally for business"}, - ) - response = provider.translate_text(request) - - assert response.translated_text == "Bonjour (formal)" - mock_request.assert_called_once() - call_args = mock_request.call_args - assert "Translate formally for business" in call_args[0][1] - - def test_translate_batch_empty(self, provider): - """Test batch translation with empty list.""" - responses = provider.translate_batch([]) - assert responses == [] - - @patch.object(OllamaTranslationProvider, "translate_text") - def test_translate_batch(self, mock_translate, provider): - """Test batch translation.""" - mock_translate.side_effect = [ - TranslationResponse(translated_text="Bonjour", provider_name="ollama"), - TranslationResponse(translated_text="Monde", provider_name="ollama"), - ] - - requests = [ - TranslationRequest(text="Hello", target_language="fr"), - TranslationRequest(text="World", target_language="fr"), - ] - responses = provider.translate_batch(requests) - - assert len(responses) == 2 - assert responses[0].translated_text == "Bonjour" - assert responses[1].translated_text == "Monde" - - -class TestOllamaErrorCodes: - """Tests for Ollama error code handling.""" - - @pytest.fixture - def provider(self): - """Create an Ollama provider with no retries for error testing.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - timeout=120, - max_retries=0, - ) - - def test_context_too_long_error(self, provider): - """Test context too long error.""" - long_text = "x" * 130000 - request = TranslationRequest(text=long_text, target_language="fr") - response = provider.translate_text(request) - - assert response.error_code == OLLAMA_CONTEXT_TOO_LONG - assert response.error is not None - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - def test_model_not_found_error(self, mock_models, provider): - """Test model not found error.""" - mock_models.return_value = ["mistral", "qwen2"] - - request = TranslationRequest(text="Hello", target_language="fr") - response = provider.translate_text(request) - - assert response.error_code == OLLAMA_MODEL_NOT_FOUND - assert "llama3" in response.error - - @patch.object(OllamaTranslationProvider, "_check_model_available") - @patch("services.providers.ollama_provider.requests") - def test_unavailable_error(self, mock_requests, mock_model_available, provider): - """Test Ollama unavailable error.""" - mock_model_available.return_value = True - mock_requests.post.side_effect = RequestsConnectionError("Connection refused") - - request = TranslationRequest(text="Hello", target_language="fr") - response = provider.translate_text(request) - - assert response.error_code == OLLAMA_UNAVAILABLE - assert "indisponible" in response.error.lower() - - -class TestOllamaHealthCheck: - """Tests for Ollama health check functionality.""" - - @pytest.fixture - def provider(self): - """Create an Ollama provider instance.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - ) - - @patch("services.providers.ollama_provider.requests") - def test_health_check_available(self, mock_requests, provider): - """Test health check when Ollama is available.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [{"name": "llama3"}, {"name": "mistral"}] - } - mock_requests.get.return_value = mock_response - - status = provider.health_check() - - assert status.name == "ollama" - assert status.available is True - assert status.latency_ms is not None - assert status.model == "llama3" - assert status.model_available is True - - @patch("services.providers.ollama_provider.requests") - def test_health_check_model_not_pulled(self, mock_requests, provider): - """Test health check when model is not pulled.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"models": [{"name": "mistral"}]} - mock_requests.get.return_value = mock_response - - status = provider.health_check() - - assert status.available is False - assert "llama3" in status.error - assert status.model == "llama3" - assert status.model_available is False - - @patch("services.providers.ollama_provider.requests") - def test_health_check_unavailable(self, mock_requests, provider): - """Test health check when Ollama is unavailable.""" - mock_requests.get.side_effect = RequestsConnectionError("Connection refused") - - status = provider.health_check() - - assert status.available is False - - @patch("services.providers.ollama_provider.requests") - def test_health_check_caching(self, mock_requests, provider): - """Test that health check results are cached (no API call when cache valid).""" - import time - from services.providers.schemas import ProviderHealthStatus - - current_time = time.time() - cached_status = ProviderHealthStatus( - name="ollama", - available=True, - latency_ms=50.0, - error=None, - last_check="2024-01-15T10:00:00Z", - ) - provider._health_cache["health_check"] = { - "value": cached_status, - "timestamp": current_time, - } - - status = provider.health_check() - - assert status.available is True - mock_requests.get.assert_not_called() - - -class TestOllamaProviderRetry: - """Tests for Ollama provider retry logic.""" - - @pytest.fixture - def provider(self): - """Create an Ollama provider with retry enabled.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - max_retries=2, - retry_delay=0.01, - ) - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - @patch.object(OllamaTranslationProvider, "_make_api_request") - def test_retry_on_timeout(self, mock_request, mock_models, provider): - """Test that timeout errors trigger retry.""" - mock_models.return_value = ["llama3"] - mock_request.side_effect = [ - OllamaProviderError(OLLAMA_TIMEOUT, "Timeout"), - "Bonjour", - ] - - request = TranslationRequest(text="Hello", target_language="fr") - response = provider.translate_text(request) - - assert response.translated_text == "Bonjour" - assert mock_request.call_count == 2 - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - @patch.object(OllamaTranslationProvider, "_make_api_request") - def test_no_retry_on_model_not_found(self, mock_request, mock_models, provider): - """Test that model not found errors do not trigger retry.""" - mock_models.return_value = ["llama3"] - mock_request.side_effect = OllamaProviderError( - OLLAMA_MODEL_NOT_FOUND, "Model not found" - ) - - request = TranslationRequest(text="Hello", target_language="fr") - provider.translate_text(request) - - assert mock_request.call_count == 1 - - @patch.object(OllamaTranslationProvider, "_fetch_available_models") - @patch.object(OllamaTranslationProvider, "_make_api_request") - def test_timeout_returns_ollama_timeout_error(self, mock_request, mock_models): - """Test that timeout without retry returns OLLAMA_TIMEOUT in response.""" - provider = OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - timeout=120, - max_retries=0, - ) - mock_models.return_value = ["llama3"] - mock_request.side_effect = OllamaProviderError( - OLLAMA_TIMEOUT, - "Délai d'attente Ollama dépassé. Réessayez avec un texte plus court.", - ) - - request = TranslationRequest(text="Hello", target_language="fr") - response = provider.translate_text(request) - - assert response.error_code == OLLAMA_TIMEOUT - assert response.error is not None - assert "Délai" in response.error or "timeout" in response.error.lower() - - -class TestOllamaProviderSingleton: - """Tests for Ollama provider singleton functions.""" - - def test_get_ollama_provider(self): - """Test get_ollama_provider creates instance with config.""" - import services.providers.ollama_provider as ollama_module - - ollama_module._provider_instance = None - - with patch( - "services.providers.config.ProvidersConfig" - ) as mock_config: - mock_config.OLLAMA_BASE_URL = "http://localhost:11434" - mock_config.OLLAMA_MODEL = "llama3" - mock_config.OLLAMA_TIMEOUT = 120 - mock_config.OLLAMA_MAX_RETRIES = 2 - mock_config.OLLAMA_RETRY_DELAY = 2.0 - - provider = ollama_module.get_ollama_provider() - - assert provider is not None - assert provider._model == "llama3" - - ollama_module._provider_instance = None - - -class TestOllamaRegistryIntegration: - """Tests for Ollama provider registry integration.""" - - def test_register_ollama_provider(self): - """Test provider registration.""" - from services.providers.registry import registry - - registry.unregister("ollama") - - with patch( - "services.providers.ollama_provider.get_ollama_provider" - ) as mock_get: - mock_provider = MagicMock() - mock_get.return_value = mock_provider - - result = register_ollama_provider() - - assert result == mock_provider - assert "ollama" in registry - registry.unregister("ollama") - - -class TestOllamaModelCheck: - """Tests for Ollama model availability checking.""" - - @pytest.fixture - def provider(self): - """Create an Ollama provider instance.""" - return OllamaTranslationProvider( - base_url="http://localhost:11434", - model="llama3", - ) - - @patch("services.providers.ollama_provider.requests") - def test_fetch_available_models(self, mock_requests, provider): - """Test fetching available models.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [ - {"name": "llama3:latest"}, - {"name": "mistral:latest"}, - ] - } - mock_requests.get.return_value = mock_response - - models = provider._fetch_available_models() - - assert "llama3:latest" in models - assert "mistral:latest" in models - - def test_check_model_available(self, provider): - """Test model availability checking.""" - import time - - provider._available_models = ["llama3:latest", "mistral:latest"] - provider._models_cache_time = time.time() - - assert provider._check_model_available("llama3") is True - assert provider._check_model_available("mistral") is True - assert provider._check_model_available("qwen2") is False diff --git a/tests/test_story_3_5_api_versioning.py b/tests/test_story_3_5_api_versioning.py index 879438f..5db3b2f 100644 --- a/tests/test_story_3_5_api_versioning.py +++ b/tests/test_story_3_5_api_versioning.py @@ -106,17 +106,6 @@ class TestAPIVersioning: response = client.get("/rate-limit/status") assert response.status_code == 404 - def test_versioned_ollama_models_accessible(self, client): - """AC1: Ollama models should be accessible under /api/v1/ollama/models""" - response = client.get("/api/v1/ollama/models") - assert response.status_code == 200 - data = response.json() - assert "models" in data - - def test_unversioned_ollama_models_returns_404(self, client): - """AC2: Unversioned /ollama/models should return 404""" - response = client.get("/ollama/models") - assert response.status_code == 404 class TestAPIVersioningAdminEndpoints: diff --git a/tests/test_translate_endpoint.py b/tests/test_translate_endpoint.py index 35a7bc0..f388c0b 100644 --- a/tests/test_translate_endpoint.py +++ b/tests/test_translate_endpoint.py @@ -771,21 +771,6 @@ class TestProviderParameter: ) assert response.status_code == 202 - def test_accepts_provider_ollama(self, authenticated_client): - """Accepts provider='ollama'""" - excel_content = create_valid_excel() - response = authenticated_client.post( - TRANSLATE_URL, - files={ - "file": ( - "test.xlsx", - io.BytesIO(excel_content), - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ) - }, - data={"target_lang": "fr", "provider": "ollama"}, - ) - assert response.status_code == 202 class TestSourceLangValidation: diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 0e0eca7..aa3edc9 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -64,6 +64,15 @@ class PDFTranslator: self._provider = provider self._font_path: Optional[str] = None self._translation_stats = {"attempted": 0, "changed": 0} + self._custom_prompt: Optional[str] = None + + def set_provider(self, provider) -> None: + """Set the translation provider.""" + self._provider = provider + + def set_custom_prompt(self, prompt: Optional[str]) -> None: + """Set custom system prompt for LLM providers.""" + self._custom_prompt = prompt def _get_font_path(self) -> Optional[str]: """Resolve a Unicode-capable TTF/OTF font file.""" @@ -811,15 +820,56 @@ class PDFTranslator: # Shared helpers # ------------------------------------------------------------------ # + def _translate_with_provider( + self, texts: List[str], target_language: str, source_language: str + ) -> List[str]: + """Translate using the TranslationProvider interface (handles old & new styles).""" + from services.providers.base import TranslationProvider as NewTranslationProvider + + is_new_style = False + if isinstance(self._provider, NewTranslationProvider): + is_new_style = True + elif hasattr(self._provider, "__class__") and self._provider.__class__.__name__ in ( + "MockTranslationProvider", + "Mock", + "MagicMock", + ): + is_new_style = True + + if is_new_style: + from services.providers.schemas import TranslationRequest + custom_prompt = getattr(self, "_custom_prompt", None) + metadata = {"custom_prompt": custom_prompt} if custom_prompt else None + + requests = [ + TranslationRequest( + text=t, + target_language=target_language, + source_language=source_language, + metadata=metadata, + ) + for t in texts + ] + responses = self._provider.translate_batch(requests) + translated = [resp.translated_text for resp in responses] + else: + translated = self._provider.translate_batch(texts, target_language, source_language) + + # Fallback: keep original text for any empty/failed result + return [ + t if (t and t.strip()) else orig + for t, orig in zip(translated, texts) + ] + def _translate_single( self, text: str, target_language: str, source_language: str ) -> str: """Translate a single text string.""" if self._provider is not None: try: - result = self._provider.translate(text, target_language, source_language) - if result and result.strip(): - return result + results = self._translate_with_provider([text], target_language, source_language) + if results and results[0].strip(): + return results[0] except Exception as e: logger.warning("provider_single_failed", error=str(e)) @@ -840,7 +890,7 @@ class PDFTranslator: translated = None if self._provider is not None: try: - translated = self._provider.translate_batch(texts, target_language, source_language) + translated = self._translate_with_provider(texts, target_language, source_language) except Exception as e: logger.warning("provider_translate_failed", error=str(e))