sepehr/office_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: revue de code, doc CODE_REVIEW, forfaits 2026, traduction LLM, providers avec modèle

Browse Source 
Made-with: Cursor
This commit is contained in:
Sepehr Ramezani
2026-03-07 11:42:58 +01:00
parent 3d37ce4582
commit 473b3e26c7
181 changed files with 30617 additions and 7170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

333
services/auth_service.py
View File

@@ -5,10 +5,13 @@ This service provides user authentication with automatic backend selection:
- If DATABASE_URL is configured: Uses PostgreSQL database
- Otherwise: Falls back to JSON file storage (development mode)
"""
import os
import secrets
import hashlib
from datetime import datetime, timedelta
import uuid
import time
from datetime import datetime, timedelta, timezone
from typing import Optional, Dict, Any
import json
from pathlib import Path
@@ -19,6 +22,7 @@ logger = logging.getLogger(__name__)
# Try to import optional dependencies
try:
import jwt
JWT_AVAILABLE = True
except ImportError:
JWT_AVAILABLE = False
@@ -26,6 +30,7 @@ except ImportError:
try:
from passlib.context import CryptContext
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
PASSLIB_AVAILABLE = True
except ImportError:
@@ -41,6 +46,7 @@ if USE_DATABASE:
from database.repositories import UserRepository
from database.connection import get_sync_session, init_db as _init_db
from database import models as db_models
DATABASE_AVAILABLE = True
logger.info("Database backend enabled for authentication")
except ImportError as e:
@@ -49,21 +55,89 @@ if USE_DATABASE:
logger.warning(f"Database modules not available: {e}. Using JSON storage.")
else:
DATABASE_AVAILABLE = False
logger.info("Using JSON file storage for authentication (DATABASE_URL not configured)")
logger.info(
"Using JSON file storage for authentication (DATABASE_URL not configured)"
)
from models.subscription import User, UserCreate, PlanType, SubscriptionStatus, PLANS
# Configuration
SECRET_KEY = os.getenv("JWT_SECRET", os.getenv("JWT_SECRET_KEY", secrets.token_urlsafe(32)))
_jwt_secret = os.getenv("JWT_SECRET", os.getenv("JWT_SECRET_KEY"))
if not _jwt_secret:
_jwt_secret = secrets.token_urlsafe(32)
logger.critical(
"SECURITY: JWT_SECRET_KEY is not configured! Using an ephemeral random key. "
"ALL JWT TOKENS WILL BE INVALIDATED ON EVERY RESTART. "
"Set JWT_SECRET_KEY in your .env file immediately."
)
SECRET_KEY = _jwt_secret
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_HOURS = 24
REFRESH_TOKEN_EXPIRE_DAYS = 30
ACCESS_TOKEN_EXPIRE_MINUTES = 15
REFRESH_TOKEN_EXPIRE_DAYS = 7
# Simple file-based storage (used when database is not configured)
USERS_FILE = Path("data/users.json")
USERS_FILE.parent.mkdir(exist_ok=True)
# Token blocklist: jti → expiry timestamp (Unix).
# Uses Redis when available (persistent across restarts), falls back to in-memory.
_revoked_jtis: dict[str, float] = {}
_redis_blocklist_client = None
def _get_blocklist_redis():
"""Return Redis client for token blocklist, or None if unavailable."""
global _redis_blocklist_client
if _redis_blocklist_client is not None:
return _redis_blocklist_client if _redis_blocklist_client is not False else None
redis_url = os.getenv("REDIS_URL", "")
if not redis_url:
_redis_blocklist_client = False
return None
try:
import redis as redis_lib
client = redis_lib.from_url(redis_url, decode_responses=True)
client.ping()
_redis_blocklist_client = client
logger.info("Token blocklist using Redis (persistent across restarts)")
return client
except Exception as e:
logger.warning(f"Redis unavailable for token blocklist, using in-memory: {e}")
_redis_blocklist_client = False
return None
def revoke_token_jti(jti: str, expires_at: float) -> None:
"""Add a JTI to the blocklist (revoked until its expiry time)."""
ttl = max(1, int(expires_at - time.time()))
redis = _get_blocklist_redis()
if redis:
try:
redis.setex(f"revoked_jti:{jti}", ttl, "1")
return
except Exception as e:
logger.warning(f"Redis revoke failed, falling back to memory: {e}")
_revoked_jtis[jti] = expires_at
def is_token_revoked(jti: str) -> bool:
"""Return True if JTI is revoked. Lazy GC of expired in-memory entries."""
if not jti:
return False
redis = _get_blocklist_redis()
if redis:
try:
return redis.exists(f"revoked_jti:{jti}") == 1
except Exception as e:
logger.warning(f"Redis revoke check failed, falling back to memory: {e}")
now = time.time()
expired = [k for k, v in _revoked_jtis.items() if v < now]
for k in expired:
_revoked_jtis.pop(k, None)
return jti in _revoked_jtis
def hash_password(password: str) -> str:
"""Hash a password using bcrypt or fallback to SHA256"""
@@ -91,34 +165,61 @@ def verify_password(plain_password: str, hashed_password: str) -> bool:
return False
def create_access_token(user_id: str, expires_delta: Optional[timedelta] = None) -> str:
"""Create a JWT access token"""
def create_access_token(
user_id: str, tier: str = "free", expires_delta: Optional[timedelta] = None
) -> str:
"""Create a JWT access token with tier claim for quick access"""
if not JWT_AVAILABLE:
# Fallback to simple token
token_data = {
"user_id": user_id,
"exp": (datetime.utcnow() + (expires_delta or timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS))).isoformat()
"tier": tier,
"exp": (
datetime.now(timezone.utc)
+ (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))
).isoformat(),
}
import base64
return base64.urlsafe_b64encode(json.dumps(token_data).encode()).decode()
expire = datetime.utcnow() + (expires_delta or timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS))
to_encode = {"sub": user_id, "exp": expire, "type": "access"}
expire = datetime.now(timezone.utc) + (
expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
)
to_encode = {
"sub": user_id,
"tier": tier,
"exp": expire,
"type": "access",
"jti": str(uuid.uuid4()),
}
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
def create_refresh_token(user_id: str) -> str:
"""Create a JWT refresh token"""
def create_refresh_token(
user_id: str, expires_delta: Optional[timedelta] = None
) -> str:
"""Create a JWT refresh token (7 days by default)"""
if not JWT_AVAILABLE:
token_data = {
"user_id": user_id,
"exp": (datetime.utcnow() + timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS)).isoformat()
"exp": (
datetime.now(timezone.utc)
+ (expires_delta or timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS))
).isoformat(),
}
import base64
return base64.urlsafe_b64encode(json.dumps(token_data).encode()).decode()
expire = datetime.utcnow() + timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS)
to_encode = {"sub": user_id, "exp": expire, "type": "refresh"}
expire = datetime.now(timezone.utc) + (
expires_delta or timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS)
)
to_encode = {
"sub": user_id,
"exp": expire,
"type": "refresh",
"jti": str(uuid.uuid4()),
}
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
@@ -127,20 +228,24 @@ def verify_token(token: str) -> Optional[Dict[str, Any]]:
if not JWT_AVAILABLE:
try:
import base64
data = json.loads(base64.urlsafe_b64decode(token.encode()).decode())
exp = datetime.fromisoformat(data["exp"])
if exp < datetime.utcnow():
if exp < datetime.now(timezone.utc):
return None
return {"sub": data["user_id"]}
except:
except Exception:
return None
try:
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
jti = payload.get("jti")
if jti and is_token_revoked(jti):
return None
return payload
except jwt.ExpiredSignatureError:
return None
except jwt.JWTError:
except jwt.PyJWTError:
return None
@@ -148,16 +253,17 @@ def load_users() -> Dict[str, Dict]:
"""Load users from file storage (JSON backend only)"""
if USERS_FILE.exists():
try:
with open(USERS_FILE, 'r') as f:
with open(USERS_FILE, "r") as f:
return json.load(f)
except:
except Exception as e:
logger.error(f"Failed to load users file: {e}")
return {}
return {}
def save_users(users: Dict[str, Dict]):
"""Save users to file storage (JSON backend only)"""
with open(USERS_FILE, 'w') as f:
with open(USERS_FILE, "w") as f:
json.dump(users, f, indent=2, default=str)
@@ -170,18 +276,21 @@ def _db_user_to_model(db_user) -> User:
password_hash=db_user.password_hash,
avatar_url=db_user.avatar_url,
plan=PlanType(db_user.plan) if db_user.plan else PlanType.FREE,
subscription_status=SubscriptionStatus(db_user.subscription_status) if db_user.subscription_status else SubscriptionStatus.ACTIVE,
subscription_status=SubscriptionStatus(db_user.subscription_status)
if db_user.subscription_status
else SubscriptionStatus.ACTIVE,
stripe_customer_id=db_user.stripe_customer_id,
stripe_subscription_id=db_user.stripe_subscription_id,
docs_translated_this_month=db_user.docs_translated_this_month or 0,
pages_translated_this_month=db_user.pages_translated_this_month or 0,
api_calls_this_month=db_user.api_calls_this_month or 0,
daily_translation_count=getattr(db_user, "daily_translation_count", 0) or 0,
extra_credits=db_user.extra_credits or 0,
usage_reset_date=db_user.usage_reset_date or datetime.utcnow(),
default_source_lang=db_user.default_source_lang or "en",
default_target_lang=db_user.default_target_lang or "es",
default_provider=db_user.default_provider or "google",
created_at=db_user.created_at or datetime.utcnow(),
usage_reset_date=db_user.usage_reset_date or datetime.now(timezone.utc),
default_source_lang=getattr(db_user, "default_source_lang", None) or "en",
default_target_lang=getattr(db_user, "default_target_lang", None) or "es",
default_provider=getattr(db_user, "default_provider", None) or "google",
created_at=db_user.created_at or datetime.now(timezone.utc),
updated_at=db_user.updated_at,
)
@@ -189,6 +298,9 @@ def _db_user_to_model(db_user) -> User:
def get_user_by_email(email: str) -> Optional[User]:
"""Get a user by email"""
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_email(email)
@@ -206,6 +318,9 @@ def get_user_by_email(email: str) -> Optional[User]:
def get_user_by_id(user_id: str) -> Optional[User]:
"""Get a user by ID"""
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_id(user_id)
@@ -224,26 +339,26 @@ def create_user(user_create: UserCreate) -> User:
# Check if email exists
if get_user_by_email(user_create.email):
raise ValueError("Email already registered")
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.create(
email=user_create.email,
name=user_create.name,
password_hash=hash_password(user_create.password),
plan=PlanType.FREE.value,
subscription_status=SubscriptionStatus.ACTIVE.value
hashed_password=hash_password(user_create.password),
tier="free",
)
session.commit()
session.refresh(db_user)
return _db_user_to_model(db_user)
else:
users = load_users()
# Generate user ID
user_id = secrets.token_urlsafe(16)
# Create user
user = User(
id=user_id,
@@ -253,11 +368,11 @@ def create_user(user_create: UserCreate) -> User:
plan=PlanType.FREE,
subscription_status=SubscriptionStatus.ACTIVE,
)
# Save to storage
users[user_id] = user.model_dump()
save_users(users)
return user
@@ -274,46 +389,55 @@ def authenticate_user(email: str, password: str) -> Optional[User]:
def update_user(user_id: str, updates: Dict[str, Any]) -> Optional[User]:
"""Update a user's data"""
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.update(user_id, updates)
db_user = repo.update(user_id, **updates)
if db_user:
session.commit()
session.refresh(db_user)
return _db_user_to_model(db_user)
return None
else:
users = load_users()
if user_id not in users:
return None
users[user_id].update(updates)
users[user_id]["updated_at"] = datetime.utcnow().isoformat()
users[user_id]["updated_at"] = datetime.now(timezone.utc).isoformat()
save_users(users)
return User(**users[user_id])
def check_usage_limits(user: User) -> Dict[str, Any]:
"""Check if user has exceeded their plan limits"""
plan = PLANS[user.plan]
# Reset usage if it's a new month
now = datetime.utcnow()
if user.usage_reset_date.month != now.month or user.usage_reset_date.year != now.year:
update_user(user.id, {
"docs_translated_this_month": 0,
"pages_translated_this_month": 0,
"api_calls_this_month": 0,
"usage_reset_date": now.isoformat() if not USE_DATABASE else now
})
now = datetime.now(timezone.utc)
if (
user.usage_reset_date.month != now.month
or user.usage_reset_date.year != now.year
):
update_user(
user.id,
{
"docs_translated_this_month": 0,
"pages_translated_this_month": 0,
"api_calls_this_month": 0,
"usage_reset_date": now.isoformat() if not USE_DATABASE else now,
},
)
user.docs_translated_this_month = 0
user.pages_translated_this_month = 0
user.api_calls_this_month = 0
docs_limit = plan["docs_per_month"]
docs_remaining = max(0, docs_limit - user.docs_translated_this_month) if docs_limit > 0 else -1
docs_remaining = (
max(0, docs_limit - user.docs_translated_this_month) if docs_limit > 0 else -1
)
return {
"can_translate": docs_remaining != 0 or user.extra_credits > 0,
"docs_used": user.docs_translated_this_month,
@@ -332,15 +456,15 @@ def record_usage(user_id: str, pages_count: int, use_credits: bool = False) -> b
user = get_user_by_id(user_id)
if not user:
return False
updates = {
"docs_translated_this_month": user.docs_translated_this_month + 1,
"pages_translated_this_month": user.pages_translated_this_month + pages_count,
}
if use_credits:
updates["extra_credits"] = max(0, user.extra_credits - pages_count)
result = update_user(user_id, updates)
return result is not None
@@ -350,11 +474,94 @@ def add_credits(user_id: str, credits: int) -> bool:
user = get_user_by_id(user_id)
if not user:
return False
result = update_user(user_id, {"extra_credits": user.extra_credits + credits})
return result is not None
# Valid plan values for admin tier change (Story 1.7)
VALID_PLAN_VALUES = {"free", "starter", "pro", "business", "enterprise"}
def update_user_plan(user_id: str, plan: str) -> Optional[User]:
"""
Update a user's plan/tier (admin only). Keeps User.plan and User.tier in sync.
tier is set to 'pro' for pro/business/enterprise, 'free' otherwise (DB constraint).
"""
plan_lower = (plan or "").strip().lower()
if plan_lower not in VALID_PLAN_VALUES:
return None
plan_enum = PlanType(plan_lower)
tier = (
"pro"
if plan_enum in (PlanType.PRO, PlanType.BUSINESS, PlanType.ENTERPRISE)
else "free"
)
if USE_DATABASE and DATABASE_AVAILABLE:
updates = {"plan": plan_enum, "tier": tier}
else:
updates = {"plan": plan_lower, "tier": tier}
return update_user(user_id, updates)
def get_user_by_api_key(api_key: str) -> Optional[User]:
"""
Get a user by API key.
Verifies that:
- The key exists in the database
- The key is active (is_active=True)
- The key hasn't expired (expires_at is None or in the future)
Returns the user associated with the API key, or None if invalid/revoked.
Raises:
ValueError: With code "API_KEY_REVOKED" if key exists but is inactive
"""
if not api_key:
return None
# Only database backend supports API keys
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.models import ApiKey
import hashlib
# Hash the provided key to compare with stored hash
key_hash = hashlib.sha256(api_key.encode()).hexdigest()
with get_sync_session() as session:
api_key_record = (
session.query(ApiKey).filter(ApiKey.key_hash == key_hash).first()
)
if not api_key_record:
return None
# Check if key is active (Story 3.2 - Revocation check)
if not api_key_record.is_active:
raise ValueError("API_KEY_REVOKED")
# Check expiration if set
if api_key_record.expires_at:
if api_key_record.expires_at < datetime.now(timezone.utc):
raise ValueError("API_KEY_EXPIRED")
# Update last_used_at and usage_count
api_key_record.last_used_at = datetime.now(timezone.utc)
api_key_record.usage_count = (api_key_record.usage_count or 0) + 1
session.commit()
# Get the user
user_id = api_key_record.user_id
return get_user_by_id(str(user_id))
return None
def init_database():
"""Initialize the database (call on application startup)"""
if USE_DATABASE and DATABASE_AVAILABLE:

77
services/auth_service_db.py
View File

@@ -2,16 +2,18 @@
Database-backed authentication service
Replaces JSON file storage with SQLAlchemy
"""
import os
import secrets
import hashlib
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from typing import Optional, Dict, Any
import logging
# Try to import optional dependencies
try:
import jwt
JWT_AVAILABLE = True
except ImportError:
JWT_AVAILABLE = False
@@ -20,6 +22,7 @@ except ImportError:
try:
from passlib.context import CryptContext
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
PASSLIB_AVAILABLE = True
except ImportError:
@@ -65,24 +68,26 @@ def verify_password(plain_password: str, hashed_password: str) -> bool:
def create_access_token(user_id: str, expires_delta: Optional[timedelta] = None) -> str:
"""Create a JWT access token"""
expire = datetime.utcnow() + (expires_delta or timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS))
expire = datetime.now(timezone.utc) + (
expires_delta or timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS)
)
if not JWT_AVAILABLE:
token_data = {"user_id": user_id, "exp": expire.isoformat(), "type": "access"}
return base64.urlsafe_b64encode(json.dumps(token_data).encode()).decode()
to_encode = {"sub": user_id, "exp": expire, "type": "access"}
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
def create_refresh_token(user_id: str) -> str:
"""Create a JWT refresh token"""
expire = datetime.utcnow() + timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS)
expire = datetime.now(timezone.utc) + timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS)
if not JWT_AVAILABLE:
token_data = {"user_id": user_id, "exp": expire.isoformat(), "type": "refresh"}
return base64.urlsafe_b64encode(json.dumps(token_data).encode()).decode()
to_encode = {"sub": user_id, "exp": expire, "type": "refresh"}
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
@@ -93,12 +98,12 @@ def verify_token(token: str) -> Optional[Dict[str, Any]]:
try:
data = json.loads(base64.urlsafe_b64decode(token.encode()).decode())
exp = datetime.fromisoformat(data["exp"])
if exp < datetime.utcnow():
if exp < datetime.now(timezone.utc):
return None
return {"sub": data["user_id"], "type": data.get("type", "access")}
except Exception:
return None
try:
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
return payload
@@ -112,18 +117,18 @@ def create_user(email: str, name: str, password: str) -> User:
"""Create a new user in the database"""
with get_db_session() as db:
repo = UserRepository(db)
# Check if email already exists
existing = repo.get_by_email(email)
if existing:
raise ValueError("Email already registered")
password_hash = hash_password(password)
hashed = hash_password(password)
user = repo.create(
email=email,
name=name,
password_hash=password_hash,
plan=PlanType.FREE,
hashed_password=hashed,
tier="free",
)
return user
@@ -133,15 +138,15 @@ def authenticate_user(email: str, password: str) -> Optional[User]:
with get_db_session() as db:
repo = UserRepository(db)
user = repo.get_by_email(email)
if not user:
return None
if not verify_password(password, user.password_hash):
return None
# Update last login
repo.update(user.id, last_login_at=datetime.utcnow())
repo.update(user.id, last_login_at=datetime.now(timezone.utc))
return user
@@ -181,11 +186,15 @@ def use_credits(user_id: str, credits: int) -> bool:
return repo.use_credits(user_id, credits)
def increment_usage(user_id: str, docs: int = 0, pages: int = 0, api_calls: int = 0) -> bool:
def increment_usage(
user_id: str, docs: int = 0, pages: int = 0, api_calls: int = 0
) -> bool:
"""Increment user usage counters"""
with get_db_session() as db:
repo = UserRepository(db)
result = repo.increment_usage(user_id, docs=docs, pages=pages, api_calls=api_calls)
result = repo.increment_usage(
user_id, docs=docs, pages=pages, api_calls=api_calls
)
return result is not None
@@ -194,12 +203,12 @@ def check_usage_limits(user_id: str) -> Dict[str, Any]:
with get_db_session() as db:
repo = UserRepository(db)
user = repo.get_by_id(user_id)
if not user:
return {"allowed": False, "reason": "User not found"}
plan_config = PLANS.get(user.plan, PLANS[PlanType.FREE])
# Check document limit
docs_limit = plan_config["docs_per_month"]
if docs_limit > 0 and user.docs_translated_this_month >= docs_limit:
@@ -211,10 +220,12 @@ def check_usage_limits(user_id: str) -> Dict[str, Any]:
"limit": docs_limit,
"used": user.docs_translated_this_month,
}
return {
"allowed": True,
"docs_remaining": max(0, docs_limit - user.docs_translated_this_month) if docs_limit > 0 else -1,
"docs_remaining": max(0, docs_limit - user.docs_translated_this_month)
if docs_limit > 0
else -1,
"extra_credits": user.extra_credits,
}
@@ -224,22 +235,28 @@ def get_user_usage_stats(user_id: str) -> Dict[str, Any]:
with get_db_session() as db:
repo = UserRepository(db)
user = repo.get_by_id(user_id)
if not user:
return {}
plan_config = PLANS.get(user.plan, PLANS[PlanType.FREE])
return {
"docs_used": user.docs_translated_this_month,
"docs_limit": plan_config["docs_per_month"],
"docs_remaining": max(0, plan_config["docs_per_month"] - user.docs_translated_this_month) if plan_config["docs_per_month"] > 0 else -1,
"docs_remaining": max(
0, plan_config["docs_per_month"] - user.docs_translated_this_month
)
if plan_config["docs_per_month"] > 0
else -1,
"pages_used": user.pages_translated_this_month,
"extra_credits": user.extra_credits,
"max_pages_per_doc": plan_config["max_pages_per_doc"],
"max_file_size_mb": plan_config["max_file_size_mb"],
"allowed_providers": plan_config["providers"],
"api_access": plan_config.get("api_access", False),
"api_calls_used": user.api_calls_this_month if plan_config.get("api_access") else 0,
"api_calls_used": user.api_calls_this_month
if plan_config.get("api_access")
else 0,
"api_calls_limit": plan_config.get("api_calls_per_month", 0),
}

183
services/glossary_service.py Normal file
View File

@@ -0,0 +1,183 @@
"""
Glossary Service for Translation
Story 3.10: Glossaires - Application lors Traduction LLM
Provides functions to retrieve glossary terms and format them for LLM prompts.
"""
import logging
from typing import List, Dict, Any, Optional
from database.connection import get_sync_session
from database.models import Glossary, GlossaryTerm
from utils.exceptions import GlossaryNotFoundError
logger = logging.getLogger(__name__)
def get_glossary_terms(glossary_id: str, user_id: str) -> List[Dict[str, str]]:
"""
Retrieve glossary terms for a specific glossary owned by a user.
Args:
glossary_id: UUID of the glossary
user_id: UUID of the user (must own the glossary)
Returns:
List of dictionaries with 'source' and 'target' keys
Raises:
GlossaryNotFoundError: If glossary doesn't exist or doesn't belong to user
"""
try:
with get_sync_session() as session:
glossary = (
session.query(Glossary)
.filter(Glossary.id == glossary_id, Glossary.user_id == user_id)
.first()
)
if not glossary:
raise GlossaryNotFoundError(
message="Glossaire introuvable ou vous n'avez pas accès à cette ressource.",
details={"glossary_id": glossary_id}
)
# Get all terms for this glossary
terms = (
session.query(GlossaryTerm)
.filter(GlossaryTerm.glossary_id == glossary_id)
.all()
)
# Format as list of dicts
result = [{"source": term.source, "target": term.target} for term in terms]
logger.info(
f"Retrieved {len(result)} terms from glossary {glossary_id} for user {user_id}"
)
return result
except GlossaryNotFoundError:
raise
except Exception as e:
logger.error(f"Error retrieving glossary {glossary_id}: {e}")
raise GlossaryNotFoundError(
message="Erreur lors de la récupération du glossaire.",
details={"glossary_id": glossary_id, "error": str(e)}
)
def validate_glossary_access(glossary_id: str, user_id: str) -> bool:
"""
Validate that a glossary exists and belongs to the user.
This is a lightweight check that doesn't return the terms,
useful for early validation before starting a translation job.
Args:
glossary_id: UUID of the glossary
user_id: UUID of the user (must own the glossary)
Returns:
True if glossary exists and belongs to user
Raises:
GlossaryNotFoundError: If glossary doesn't exist or doesn't belong to user
"""
try:
with get_sync_session() as session:
glossary = (
session.query(Glossary)
.filter(Glossary.id == glossary_id, Glossary.user_id == user_id)
.first()
)
if not glossary:
raise GlossaryNotFoundError(
message="Glossaire introuvable ou vous n'avez pas accès à cette ressource.",
details={"glossary_id": glossary_id}
)
return True
except GlossaryNotFoundError:
raise
except Exception as e:
logger.error(f"Error validating glossary access {glossary_id}: {e}")
raise GlossaryNotFoundError(
message="Erreur lors de la validation du glossaire.",
details={"glossary_id": glossary_id, "error": str(e)}
)
def format_glossary_for_prompt(terms: List[Dict[str, str]]) -> str:
"""
Format glossary terms for injection into an LLM system prompt.
The format is designed to be clear and unambiguous for LLMs:
- Clear header explaining the purpose
- Simple source → target format
- Explicit instruction to use these translations
Args:
terms: List of dictionaries with 'source' and 'target' keys
Returns:
Formatted string for LLM prompt
"""
if not terms:
return ""
# Sort terms by length (longest first) to avoid substring conflicts
# e.g., "machine learning" should match before "machine"
sorted_terms = sorted(terms, key=lambda t: len(t.get("source", "")), reverse=True)
lines = [
"TERMINOLOGY GLOSSARY (use these exact translations):",
""
]
for term in sorted_terms:
source = term.get("source", "").strip()
target = term.get("target", "").strip()
if source and target:
# Escape single quotes in terms for clarity
source_escaped = source.replace("'", "\\'")
target_escaped = target.replace("'", "\\'")
lines.append(f"- '{source_escaped}' → '{target_escaped}'")
lines.extend([
"",
"IMPORTANT: Always use these translations when the terms appear in the text."
])
return "\n".join(lines)
def build_full_prompt(
custom_prompt: Optional[str],
glossary_terms: Optional[List[Dict[str, str]]]
) -> str:
"""
Build the complete prompt combining custom prompt and glossary.
Args:
custom_prompt: Optional custom system prompt from user
glossary_terms: Optional list of glossary terms
Returns:
Combined prompt string
"""
parts = []
if custom_prompt:
parts.append(custom_prompt)
if glossary_terms:
glossary_prompt = format_glossary_for_prompt(glossary_terms)
if glossary_prompt:
parts.append(glossary_prompt)
return "\n\n".join(parts) if parts else ""

174
services/progress_tracker.py Normal file
View File

@@ -0,0 +1,174 @@
"""
Progress Tracker Service (Story 2.11)
Provides real-time progress tracking for translation jobs.
Designed for O(1) updates and < 500ms latency (NFR3).
"""
from typing import Dict, Any, Optional, Callable
import threading
import time
class ProgressTracker:
"""
Track translation progress with callback support.
Designed for high-performance updates with minimal overhead.
Uses in-memory storage for MVP (consistent with Story 2.10 pattern).
Usage:
storage = {} # Reference to _translation_jobs dict
tracker = ProgressTracker("job_123", storage)
tracker.update(50, "Translating sheet 2/4")
# Or use item-based progress
tracker.update_item(3, 10, "Translating slide")
"""
def __init__(self, job_id: str, storage: Dict[str, Any]):
"""
Initialize progress tracker.
Args:
job_id: The translation job ID
storage: Reference to the job storage dict (e.g., _translation_jobs)
"""
self.job_id = job_id
self.storage = storage
self._lock = threading.RLock()
self._last_update_time = 0
self._min_update_interval = 0.05 # 50ms minimum between updates (throttling)
def update(self, percent: int, step: str) -> None:
"""
Update progress percentage and current step.
Thread-safe and throttled to prevent excessive updates.
Args:
percent: Progress percentage (0-100), will be clamped
step: Human-readable description of current operation
"""
with self._lock:
current_time = time.time()
if current_time - self._last_update_time < self._min_update_interval:
if percent < 100:
return
job = self.storage.get(self.job_id)
if job:
# Never decrease progress — only move forward.
new_percent = min(100, max(0, percent))
job["progress_percent"] = max(job.get("progress_percent", 0), new_percent)
job["current_step"] = step
job["processed_items"] = job.get("processed_items", 0)
job["total_items"] = job.get("total_items", 0)
self._last_update_time = current_time
def update_item(
self, current: int, total: int, item_name: str, max_percent: int = 100
) -> None:
"""
Update progress based on item count (e.g., slides, sheets).
Calculates percentage from current/total and formats step message.
Args:
current: Current item number (1-based)
total: Total number of items
item_name: Name of item type (e.g., "Translating slide", "Processing sheet")
max_percent: Upper bound for the computed percentage (default 100).
Use 95 to reserve the last 5% for file-save + set_completed().
"""
percent = int((current / total) * 100) if total > 0 else 0
percent = min(percent, max_percent)
step = f"{item_name} {current}/{total}"
with self._lock:
current_time = time.time()
if current_time - self._last_update_time < self._min_update_interval:
if percent < 100:
return
job = self.storage.get(self.job_id)
if job:
# Never decrease progress — only move forward.
new_percent = min(100, max(0, percent))
job["progress_percent"] = max(job.get("progress_percent", 0), new_percent)
job["current_step"] = step
job["processed_items"] = current
job["total_items"] = total
self._last_update_time = current_time
def set_error(
self, error_message: str, step: str = "Error during translation"
) -> None:
"""
Mark job as failed with error message.
Args:
error_message: Description of the error
step: Current step description (default: "Error during translation")
"""
with self._lock:
job = self.storage.get(self.job_id)
if job:
job["status"] = "failed"
job["error_message"] = error_message
job["current_step"] = step
job["failed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def set_completed(self, output_path: Optional[str] = None) -> None:
"""
Mark job as completed.
Args:
output_path: Optional path to the output file
"""
with self._lock:
job = self.storage.get(self.job_id)
if job:
job["status"] = "completed"
job["progress_percent"] = 100
job["current_step"] = "Translation complete"
job["completed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
if output_path:
job["output_path"] = str(output_path)
def create_progress_callback(
tracker: ProgressTracker, item_name: str, total_items: int
) -> Callable[[Dict[str, Any]], None]:
"""
Create a progress callback function for use with translators.
Args:
tracker: ProgressTracker instance
item_name: Name of item being processed (e.g., "Translating slide")
total_items: Total number of items
Returns:
Callback function compatible with translator progress_callback parameter
"""
def callback(progress_info: Dict[str, Any]) -> None:
"""Progress callback that updates the tracker."""
# Extract item number from progress_info dict
# Different translators use different keys
current = progress_info.get(
"slide",
progress_info.get(
"sheet", progress_info.get("paragraph", progress_info.get("element", 1))
),
)
total = progress_info.get(
"total_slides",
progress_info.get(
"total", progress_info.get("total_paragraphs", total_items)
),
)
tracker.update_item(current, total, item_name)
return callback

127
services/prompt_service.py Normal file
View File

@@ -0,0 +1,127 @@
"""
Prompt Service for Translation
Story 3.12: Custom Prompts - Application lors Traduction LLM
Provides functions to retrieve prompt content and validate access.
"""
import uuid
import logging
from typing import Optional, Tuple
from database.connection import get_sync_session
from database.models import CustomPrompt
from utils.exceptions import PromptNotFoundError
logger = logging.getLogger(__name__)
def _validate_uuid(id_str: str, id_name: str = "ID") -> None:
"""
Validate that a string is a valid UUID.
Args:
id_str: String to validate
id_name: Name of the ID for error messages
Raises:
PromptNotFoundError: If the string is not a valid UUID
"""
try:
uuid.UUID(id_str)
except (ValueError, AttributeError):
raise PromptNotFoundError(
message=f"{id_name} invalide.",
details={id_name.lower(): id_str}
)
def _get_prompt_record(prompt_id: str, user_id: str) -> Tuple[CustomPrompt, bool]:
"""
Internal helper to fetch a prompt record from the database.
This is a shared function to avoid code duplication between
get_prompt_content and validate_prompt_access.
Args:
prompt_id: UUID of the prompt
user_id: UUID of the user (must own the prompt)
Returns:
Tuple of (CustomPrompt, was_logged) - was_logged indicates if access was already logged
Raises:
PromptNotFoundError: If prompt doesn't exist or doesn't belong to user
"""
# Validate UUIDs before querying database
_validate_uuid(prompt_id, "prompt_id")
_validate_uuid(user_id, "user_id")
try:
with get_sync_session() as session:
prompt = (
session.query(CustomPrompt)
.filter(CustomPrompt.id == prompt_id, CustomPrompt.user_id == user_id)
.first()
)
if not prompt:
raise PromptNotFoundError(
message="Prompt introuvable ou vous n'avez pas accès à cette ressource.",
details={"prompt_id": prompt_id}
)
return prompt, False
except PromptNotFoundError:
raise
except Exception as e:
logger.error(f"Error fetching prompt {prompt_id}: {e}")
raise PromptNotFoundError(
message="Erreur lors de la récupération du prompt.",
details={"prompt_id": prompt_id, "error": str(e)}
)
def get_prompt_content(prompt_id: str, user_id: str) -> str:
"""
Retrieve prompt content for a specific prompt owned by a user.
Args:
prompt_id: UUID of the prompt
user_id: UUID of the user (must own the prompt)
Returns:
The prompt content string
Raises:
PromptNotFoundError: If prompt doesn't exist or doesn't belong to user
"""
prompt, _ = _get_prompt_record(prompt_id, user_id)
logger.info(
f"Retrieved prompt '{prompt.name}' ({prompt_id}) for user {user_id}"
)
return prompt.content
def validate_prompt_access(prompt_id: str, user_id: str) -> bool:
"""
Validate that a prompt exists and belongs to the user.
Lightweight check before starting a translation job.
Does NOT log to avoid duplicate log entries when followed by get_prompt_content.
Args:
prompt_id: UUID of the prompt
user_id: UUID of the user (must own the prompt)
Returns:
True if prompt exists and belongs to user
Raises:
PromptNotFoundError: If prompt doesn't exist or doesn't belong to user
"""
_get_prompt_record(prompt_id, user_id)
return True

282
services/providers/README.md Normal file
View File

@@ -0,0 +1,282 @@
# Translation Providers
This directory contains translation provider implementations for the office_translator service.
## Available Providers
### Google Translate (`google_provider.py`)
Production-ready Google Translate provider with:
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Health check with result caching (60s TTL)
- Usage metrics logging
**Configuration:**
```bash
GOOGLE_TRANSLATE_ENABLED=true
GOOGLE_TRANSLATE_TIMEOUT=30
GOOGLE_TRANSLATE_MAX_RETRIES=3
GOOGLE_TRANSLATE_RETRY_DELAY=1
```
**API Usage:**
- Free tier: 500,000 characters/month
- 5,000 characters max per request
- Cost: ~$20 per million characters (paid tier)
**Error Codes:**
| Code | Description |
|------|-------------|
| `GOOGLE_QUOTA_EXCEEDED` | API quota exceeded (429) |
| `GOOGLE_INVALID_KEY` | Invalid API key (401/403) |
| `GOOGLE_NETWORK_ERROR` | Network/timeout error (502) |
| `GOOGLE_UNSUPPORTED_LANGUAGE` | Language not supported (400) |
| `GOOGLE_TEXT_TOO_LONG` | Text exceeds 5000 chars (413) |
### DeepL (`deepl_provider.py`)
Production-ready DeepL provider with:
- Automatic Free/Pro endpoint detection based on API key format
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Health check with result caching (60s TTL)
- Language code normalization for DeepL compatibility
**Configuration:**
```bash
DEEPL_ENABLED=true
DEEPL_API_KEY=your_deepl_api_key_here # Free keys end with :fx
DEEPL_TIMEOUT=30
DEEPL_MAX_RETRIES=3
DEEPL_RETRY_DELAY=1
```
**Free vs Pro API Keys:**
| Type | Key Format | Endpoint |
|------|------------|----------|
| Free | Ends with `:fx` | `https://api-free.deepl.com/v2/translate` |
| Pro | Does NOT end with `:fx` | `https://api.deepl.com/v2/translate` |
**API Usage:**
- Free tier: 500,000 characters/month
- Pro tier: ~€25 per million characters
- 128KB max per request
- Higher quality for European languages
**Supported Languages:**
BG, CS, DA, DE, EL, EN-GB, EN-US, ES, ET, FI, FR, HU, ID, IT, JA, KO, LT, LV, NB, NL, PL, PT-BR, PT-PT, RO, RU, SK, SL, SV, TR, UK, ZH
**Language Notes:**
- English has two variants: EN-GB, EN-US (defaults to EN-US)
- Portuguese has two variants: PT-BR, PT-PT (defaults to PT-BR)
- Language codes are case-sensitive (uppercase)
- Auto-detect uses `auto` (like Google)
**Error Codes:**
| Code | HTTP | Description |
|------|------|-------------|
| `DEEPL_QUOTA_EXCEEDED` | 429 | Character quota exceeded |
| `DEEPL_INVALID_KEY` | 401 | Invalid API key |
| `DEEPL_NETWORK_ERROR` | 502 | Network/timeout error |
| `DEEPL_UNSUPPORTED_LANGUAGE` | 400 | Language not supported |
| `DEEPL_TEXT_TOO_LONG` | 413 | Text exceeds 128KB |
### OpenAI (`openai_provider.py`)
Cloud LLM translation provider with:
- GPT-4/GPT-4o/GPT-4o-mini model support
- Custom system prompt support for translation context
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Fast timeout for cloud API (default 60s)
- Health check with result caching (60s TTL)
**Configuration:**
```bash
OPENAI_ENABLED=true
OPENAI_API_KEY=sk-proj-xxxxxxxxxxxxxxxxxxxxxxxx
OPENAI_MODEL=gpt-4o-mini
OPENAI_TIMEOUT=60
OPENAI_MAX_RETRIES=3
OPENAI_RETRY_DELAY=1.0
# OPENAI_BASE_URL=https://api.openai.com/v1 # Optional: for Azure OpenAI or proxies
```
**Prerequisites:**
- OpenAI API key from https://platform.openai.com/api-keys
- Valid billing method on your OpenAI account
**Recommended Models for Translation:**
| Model | Cost | Speed | Quality | Best For |
|-------|------|-------|---------|----------|
| `gpt-4o-mini` | $0.15/M tokens | Fast | Good | Default choice, cost-effective |
| `gpt-4o` | $2.50/M tokens | Medium | Excellent | High-quality requirements |
| `gpt-4` | $30/M tokens | Slower | Excellent | Critical translations |
| `gpt-3.5-turbo` | $0.50/M tokens | Fastest | Good | Speed priority |
**Custom System Prompt:**
```python
request = TranslationRequest(
text="Hello",
target_language="fr",
metadata={"custom_prompt": "Translate formally for business context"}
)
```
**Rate Limiting:**
- OpenAI has strict rate limits per tier
- The provider automatically handles 429 errors with retry
- Retry-After header is respected when available
- Exponential backoff for transient errors
**Error Codes:**
| Code | HTTP | Description |
|------|------|-------------|
| `OPENAI_RATE_LIMITED` | 429 | Rate limit hit, retry suggested |
| `OPENAI_INVALID_KEY` | 401 | Invalid API key |
| `OPENAI_QUOTA_EXCEEDED` | 429 | Billing quota exceeded |
| `OPENAI_TIMEOUT` | 502 | Request timeout |
| `OPENAI_SERVICE_ERROR` | 502 | OpenAI server error |
| `OPENAI_CONTEXT_TOO_LONG` | 413 | Text exceeds model limit |
### Ollama (`ollama_provider.py`)
Local LLM translation provider with:
- Custom system prompt support for translation context
- Automatic model availability checking
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Longer timeout for LLM operations (default 120s)
- Health check with result caching (60s TTL)
**Configuration:**
```bash
OLLAMA_ENABLED=true
OLLAMA_BASE_URL=http://localhost:11434
OLLAMA_MODEL=llama3
OLLAMA_VISION_MODEL=llava
OLLAMA_TIMEOUT=120
OLLAMA_MAX_RETRIES=2
OLLAMA_RETRY_DELAY=2
```
**Prerequisites:**
- Ollama must be installed and running: `ollama serve`
- Model must be pulled before use: `ollama pull llama3`
**Recommended Models for Translation:**
| Model | Size | Best For |
|-------|------|----------|
| `llama3` | 8B | General translation, good balance |
| `llama3:70b` | 70B | High-quality translation |
| `mistral` | 7B | Fast translation |
| `qwen2` | 7B | Strong non-English support |
**Custom System Prompt:**
```python
request = TranslationRequest(
text="Hello",
target_language="fr",
metadata={"custom_prompt": "Translate formally for business context"}
)
```
**Error Codes:**
| Code | HTTP | Description |
|------|------|-------------|
| `OLLAMA_UNAVAILABLE` | 502 | Ollama service not reachable |
| `OLLAMA_MODEL_NOT_FOUND` | 400 | Model not pulled |
| `OLLAMA_TIMEOUT` | 502 | Request timeout |
| `OLLAMA_GENERATION_ERROR` | 502 | LLM generation failed |
| `OLLAMA_CONTEXT_TOO_LONG` | 413 | Text exceeds model limit |
## Usage
```python
from services.providers.google_provider import GoogleTranslationProvider
from services.providers.deepl_provider import DeepLTranslationProvider
from services.providers.openai_provider import OpenAITranslationProvider
from services.providers.ollama_provider import OllamaTranslationProvider
from services.providers.schemas import TranslationRequest
# Google provider
google_provider = GoogleTranslationProvider()
request = TranslationRequest(text="Hello", target_language="fr")
response = google_provider.translate_text(request)
# DeepL provider (requires API key)
deepl_provider = DeepLTranslationProvider(api_key="your-key:fx")
request = TranslationRequest(text="Hello", target_language="fr")
response = deepl_provider.translate_text(request)
# OpenAI provider (requires API key)
openai_provider = OpenAITranslationProvider(
api_key="sk-...",
model="gpt-4o-mini"
)
request = TranslationRequest(text="Hello", target_language="fr")
response = openai_provider.translate_text(request)
# OpenAI with custom prompt
request = TranslationRequest(
text="Hello",
target_language="fr",
metadata={"custom_prompt": "Translate formally for business context"}
)
response = openai_provider.translate_text(request)
# Ollama provider (requires local Ollama running)
ollama_provider = OllamaTranslationProvider(
base_url="http://localhost:11434",
model="llama3"
)
request = TranslationRequest(text="Hello", target_language="fr")
response = ollama_provider.translate_text(request)
# Ollama with custom prompt
request = TranslationRequest(
text="Hello",
target_language="fr",
metadata={"custom_prompt": "Translate formally"}
)
response = ollama_provider.translate_text(request)
if response.success:
print(response.translated_text)
else:
print(f"Error: {response.error_code} - {response.error}")
```
## Registry Usage
```python
from services.providers import registry
# List all providers
print(registry.list_all())
# Get first available from fallback chain
provider = registry.get_first_available(["google", "deepl", "openai", "ollama"])
# Check if provider is available
print(registry.list_available())
```
## Health Check
```python
status = provider.health_check()
print(f"Available: {status.available}")
print(f"Latency: {status.latency_ms}ms")
print(f"Last Check: {status.last_check}")
```
## Architecture
All providers extend `TranslationProvider` base class and implement:
- `translate_text(request: TranslationRequest) -> TranslationResponse`
- `translate_batch(requests: List[TranslationRequest]) -> List[TranslationResponse]`
- `is_available() -> bool`
- `health_check() -> ProviderHealthStatus`
- `get_name() -> str`

81
services/providers/__init__.py Normal file
View File

@@ -0,0 +1,81 @@
"""
Translation Providers Package.
This package provides a pluggable architecture for translation providers
with a registry for easy access and fallback support.
Usage:
from services.providers import TranslationProvider, registry
from services.providers.schemas import TranslationRequest, TranslationResponse
# Get a provider (Google is auto-registered)
google_provider = registry.get("google")
# Translate text
request = TranslationRequest(text="Hello", target_language="fr")
response = google_provider.translate_text(request)
# Use fallback chain
provider = registry.get_first_available(["google", "deepl", "openai"])
"""
from .base import TranslationProvider
from .schemas import (
TranslationRequest,
TranslationResponse,
BatchTranslationRequest,
BatchTranslationResponse,
ProviderHealthStatus,
)
from .registry import ProviderRegistry, registry, get_registry
__all__ = [
"TranslationProvider",
"TranslationRequest",
"TranslationResponse",
"BatchTranslationRequest",
"BatchTranslationResponse",
"ProviderHealthStatus",
"ProviderRegistry",
"registry",
"get_registry",
"translate_with_fallback",
"translate_with_fallback_by_mode",
"AllProvidersFailedError",
"ALL_PROVIDERS_FAILED",
]
def _auto_register_providers() -> None:
"""Auto-register available providers on module import."""
from .google_provider import register_google_provider
from .config import ProvidersConfig
if ProvidersConfig.GOOGLE_ENABLED:
register_google_provider()
if ProvidersConfig.DEEPL_ENABLED and ProvidersConfig.DEEPL_API_KEY:
from .deepl_provider import register_deepl_provider
register_deepl_provider()
if ProvidersConfig.OLLAMA_ENABLED:
from .ollama_provider import register_ollama_provider
register_ollama_provider()
if ProvidersConfig.OPENAI_ENABLED and ProvidersConfig.OPENAI_API_KEY:
from .openai_provider import register_openai_provider
register_openai_provider()
_auto_register_providers()
# Import fallback functions for easy access
from .fallback import (
translate_with_fallback,
translate_with_fallback_by_mode,
AllProvidersFailedError,
ALL_PROVIDERS_FAILED,
)

104
services/providers/base.py Normal file
View File

@@ -0,0 +1,104 @@
"""
Abstract base class for translation providers.
Provides a common interface for all translation provider implementations.
"""
from abc import ABC, abstractmethod
from typing import Optional, List
import time
from .schemas import (
TranslationRequest,
TranslationResponse,
BatchTranslationRequest,
BatchTranslationResponse,
ProviderHealthStatus,
)
class TranslationProvider(ABC):
"""
Abstract base class for translation providers.
All translation providers must implement this interface to ensure
consistent behavior across different translation services.
"""
@abstractmethod
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
"""
Translate a single text string.
Args:
request: TranslationRequest containing text, target_language, and source_language
Returns:
TranslationResponse with translated text and metadata
"""
pass
@abstractmethod
def get_name(self) -> str:
"""
Return the provider name for logging and registry.
Returns:
Provider name as a string (e.g., "google", "deepl", "openai")
"""
pass
@abstractmethod
def is_available(self) -> bool:
"""
Check if the provider is configured and reachable.
Returns:
True if the provider can perform translations, False otherwise
"""
pass
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts. Default implementation uses individual calls.
Subclasses can override this for optimized batch processing.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects in the same order as requests
"""
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
"""
Return health status details for the provider.
Performs a lightweight check to verify the provider is operational.
Returns:
ProviderHealthStatus with availability and latency information
"""
start_time = time.time()
try:
available = self.is_available()
latency_ms = (time.time() - start_time) * 1000
return ProviderHealthStatus(
name=self.get_name(),
available=available,
latency_ms=round(latency_ms, 2),
error=None if available else "Provider not available",
)
except Exception as e:
latency_ms = (time.time() - start_time) * 1000
return ProviderHealthStatus(
name=self.get_name(),
available=False,
latency_ms=round(latency_ms, 2),
error=str(e),
)

208
services/providers/config.py Normal file
View File

@@ -0,0 +1,208 @@
"""
Provider Configuration - Environment-based settings for translation providers.
Loads API keys, URLs, and enable/disable flags from environment variables.
"""
import os
from typing import List, Optional
from pydantic import BaseModel
def _ensure_dotenv_loaded() -> None:
"""Load .env file if not already loaded."""
from dotenv import load_dotenv
load_dotenv()
_ensure_dotenv_loaded()
class ProviderSettings(BaseModel):
"""Settings for a single translation provider."""
enabled: bool = False
api_key: Optional[str] = None
base_url: Optional[str] = None
model: Optional[str] = None
class ProvidersConfig:
"""
Configuration for all translation providers.
Loads settings from environment variables with sensible defaults.
"""
# Google Translate (no API key required via deep_translator)
GOOGLE_ENABLED: bool = (
os.getenv("GOOGLE_TRANSLATE_ENABLED", "true").lower() == "true"
)
GOOGLE_TRANSLATE_TIMEOUT: int = int(os.getenv("GOOGLE_TRANSLATE_TIMEOUT", "30"))
GOOGLE_TRANSLATE_MAX_RETRIES: int = int(
os.getenv("GOOGLE_TRANSLATE_MAX_RETRIES", "3")
)
GOOGLE_TRANSLATE_RETRY_DELAY: float = float(
os.getenv("GOOGLE_TRANSLATE_RETRY_DELAY", "1.0")
)
# DeepL
DEEPL_ENABLED: bool = os.getenv("DEEPL_ENABLED", "false").lower() == "true"
DEEPL_API_KEY: str = os.getenv("DEEPL_API_KEY", "")
DEEPL_TIMEOUT: int = int(os.getenv("DEEPL_TIMEOUT", "30"))
DEEPL_MAX_RETRIES: int = int(os.getenv("DEEPL_MAX_RETRIES", "3"))
DEEPL_RETRY_DELAY: float = float(os.getenv("DEEPL_RETRY_DELAY", "1.0"))
# OpenAI
OPENAI_ENABLED: bool = os.getenv("OPENAI_ENABLED", "false").lower() == "true"
OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "")
OPENAI_MODEL: str = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
OPENAI_TIMEOUT: int = int(os.getenv("OPENAI_TIMEOUT", "60"))
OPENAI_MAX_RETRIES: int = int(os.getenv("OPENAI_MAX_RETRIES", "3"))
OPENAI_RETRY_DELAY: float = float(os.getenv("OPENAI_RETRY_DELAY", "1.0"))
OPENAI_BASE_URL: str = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
OPENAI_HEALTH_CHECK_TIMEOUT: int = int(
os.getenv("OPENAI_HEALTH_CHECK_TIMEOUT", "5")
)
# Ollama (local LLM) - default model is config-only, no hardcode in provider
_DEFAULT_OLLAMA_MODEL: str = "llama3"
OLLAMA_ENABLED: bool = os.getenv("OLLAMA_ENABLED", "false").lower() == "true"
OLLAMA_BASE_URL: str = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
OLLAMA_MODEL: str = os.getenv("OLLAMA_MODEL", _DEFAULT_OLLAMA_MODEL)
OLLAMA_VISION_MODEL: str = os.getenv("OLLAMA_VISION_MODEL", "llava")
OLLAMA_TIMEOUT: int = int(os.getenv("OLLAMA_TIMEOUT", "120"))
OLLAMA_MAX_RETRIES: int = int(os.getenv("OLLAMA_MAX_RETRIES", "2"))
OLLAMA_RETRY_DELAY: float = float(os.getenv("OLLAMA_RETRY_DELAY", "2.0"))
# OpenRouter (multi-model API)
OPENROUTER_ENABLED: bool = (
os.getenv("OPENROUTER_ENABLED", "false").lower() == "true"
)
OPENROUTER_API_KEY: str = os.getenv("OPENROUTER_API_KEY", "")
OPENROUTER_MODEL: str = os.getenv("OPENROUTER_MODEL", "deepseek/deepseek-chat")
# Fallback chain configuration
# General fallback chain (backward compatibility)
FALLBACK_CHAIN: List[str] = [
name.strip()
for name in os.getenv(
"PROVIDER_FALLBACK_CHAIN", "google,deepl,openai,ollama,openrouter"
).split(",")
if name.strip()
]
# Mode-specific fallback chains
# Classic mode: Google Translate -> DeepL
FALLBACK_CHAIN_CLASSIC: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_CLASSIC", "google,deepl").split(",")
if name.strip()
]
# LLM mode: Ollama (local) -> OpenAI (cloud)
FALLBACK_CHAIN_LLM: List[str] = [
name.strip()
for name in os.getenv("FALLBACK_CHAIN_LLM", "ollama,openai").split(",")
if name.strip()
]
@classmethod
def get_fallback_chain(cls, mode: str = "auto") -> List[str]:
"""
Get the fallback chain for a specific mode.
Args:
mode: "classic" for Classic providers, "llm" for LLM providers,
"auto" or any other value for general fallback chain
Returns:
List of provider names in fallback order
"""
mode = mode.lower()
if mode == "classic":
return cls.FALLBACK_CHAIN_CLASSIC
elif mode == "llm":
return cls.FALLBACK_CHAIN_LLM
else:
return cls.FALLBACK_CHAIN
@classmethod
def get_provider_settings(cls, provider_name: str) -> ProviderSettings:
"""
Get settings for a specific provider.
Args:
provider_name: Name of the provider (e.g., "google", "deepl")
Returns:
ProviderSettings for the requested provider
"""
settings_map = {
"google": ProviderSettings(
enabled=cls.GOOGLE_ENABLED, api_key=None, base_url=None, model=None
),
"deepl": ProviderSettings(
enabled=cls.DEEPL_ENABLED,
api_key=cls.DEEPL_API_KEY if cls.DEEPL_API_KEY else None,
base_url=None,
model=None,
),
"openai": ProviderSettings(
enabled=cls.OPENAI_ENABLED,
api_key=cls.OPENAI_API_KEY if cls.OPENAI_API_KEY else None,
base_url=cls.OPENAI_BASE_URL or None,
model=cls.OPENAI_MODEL,
),
"ollama": ProviderSettings(
enabled=cls.OLLAMA_ENABLED,
api_key=None,
base_url=cls.OLLAMA_BASE_URL,
model=cls.OLLAMA_MODEL,
),
"openrouter": ProviderSettings(
enabled=cls.OPENROUTER_ENABLED,
api_key=cls.OPENROUTER_API_KEY if cls.OPENROUTER_API_KEY else None,
base_url="https://openrouter.ai/api/v1",
model=cls.OPENROUTER_MODEL,
),
}
return settings_map.get(provider_name.lower(), ProviderSettings())
@classmethod
def is_provider_configured(cls, provider_name: str) -> bool:
"""
Check if a provider is properly configured.
Args:
provider_name: Name of the provider
Returns:
True if the provider is enabled and has required configuration
"""
settings = cls.get_provider_settings(provider_name)
if not settings.enabled:
return False
# Providers requiring API keys
providers_requiring_key = {"deepl", "openai", "openrouter"}
if provider_name.lower() in providers_requiring_key:
return bool(settings.api_key)
return True
@classmethod
def get_available_providers(cls) -> List[str]:
"""
Get list of configured and available providers.
Returns:
List of provider names that are ready to use
"""
return [name for name in cls.FALLBACK_CHAIN if cls.is_provider_configured(name)]
providers_config = ProvidersConfig()

763
services/providers/deepl_provider.py Normal file
View File

@@ -0,0 +1,763 @@
"""
DeepL Provider - Production-ready implementation.
Extends TranslationProvider base class with robust error handling,
retry logic, and health monitoring.
Features:
- Automatic Free/Pro endpoint detection based on API key format
- Specific error codes for all DeepL API errors
- Retry logic with exponential backoff for transient errors
- Timeout configuration
- Health check with caching
- Structlog-compatible logging (no document content in logs)
"""
import os
import socket
import threading
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
_HAS_STRUCTLOG = True
logger = structlog.get_logger(__name__)
except ImportError:
import logging
_HAS_STRUCTLOG = False
logger = logging.getLogger(__name__)
def _log_info(event: str, **kwargs):
"""Log info message compatible with both structlog and standard logging."""
if _HAS_STRUCTLOG:
logger.info(event, **kwargs)
else:
logger.info(f"{event} {' '.join(f'{k}={v}' for k, v in kwargs.items())}")
def _log_warning(event: str, **kwargs):
"""Log warning message compatible with both structlog and standard logging."""
if _HAS_STRUCTLOG:
logger.warning(event, **kwargs)
else:
logger.warning(f"{event} {' '.join(f'{k}={v}' for k, v in kwargs.items())}")
def _log_error(event: str, **kwargs):
"""Log error message compatible with both structlog and standard logging."""
if _HAS_STRUCTLOG:
logger.error(event, **kwargs)
else:
logger.error(f"{event} {' '.join(f'{k}={v}' for k, v in kwargs.items())}")
from .base import TranslationProvider
from .schemas import (
BatchTranslationRequest,
BatchTranslationResponse,
ProviderHealthStatus,
TranslationRequest,
TranslationResponse,
)
DEEPL_QUOTA_EXCEEDED = "DEEPL_QUOTA_EXCEEDED"
DEEPL_INVALID_KEY = "DEEPL_INVALID_KEY"
DEEPL_NETWORK_ERROR = "DEEPL_NETWORK_ERROR"
DEEPL_UNSUPPORTED_LANGUAGE = "DEEPL_UNSUPPORTED_LANGUAGE"
DEEPL_TEXT_TOO_LONG = "DEEPL_TEXT_TOO_LONG"
_RETRYABLE_ERRORS = {DEEPL_NETWORK_ERROR, DEEPL_QUOTA_EXCEEDED}
DEEPL_FREE_SUFFIX = ":fx"
MAX_TEXT_LENGTH = 128 * 1024
DEEPL_SUPPORTED_LANGUAGES = {
"BG",
"CS",
"DA",
"DE",
"EL",
"EN-GB",
"EN-US",
"ES",
"ET",
"FI",
"FR",
"HU",
"ID",
"IT",
"JA",
"KO",
"LT",
"LV",
"NB",
"NL",
"PL",
"PT-BR",
"PT-PT",
"RO",
"RU",
"SK",
"SL",
"SV",
"TR",
"UK",
"ZH",
}
class DeepLProviderError(Exception):
"""Exception raised for DeepL API errors."""
def __init__(
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
):
self.code = code
self.message = message
self.details = details or {}
super().__init__(message)
def to_dict(self) -> Dict[str, Any]:
"""Convert error to dictionary format."""
result = {
"error": self.code,
"message": self.message,
}
if self.details:
result["details"] = self.details
return result
class DeepLTranslationProvider(TranslationProvider):
"""
DeepL implementation using deep_translator library.
Features:
- Automatic Free/Pro endpoint detection based on API key format
- Thread-safe translator instances per thread
- Caching support (uses global cache from translation_service)
- Batch translation with optimized processing
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Configurable timeout
- Health check with result caching
"""
def __init__(
self,
api_key: str,
use_cache: bool = True,
timeout: int = 30,
max_retries: int = 3,
retry_delay: float = 1.0,
):
"""
Initialize DeepL provider.
Args:
api_key: DeepL API key (Free keys end with :fx)
use_cache: Whether to use translation caching (default: True)
timeout: Request timeout in seconds (default: 30)
max_retries: Maximum retry attempts for transient errors (default: 3)
retry_delay: Initial retry delay in seconds (default: 1.0)
"""
if not api_key:
raise ValueError("DeepL API key is required")
self._api_key = api_key
self._api_type = self._detect_api_type(api_key)
self._local = threading.local()
self._use_cache = use_cache
self._provider_name = "deepl"
self._cache = None
self.timeout = timeout
self.max_retries = max_retries
self.retry_delay = retry_delay
self._health_cache: Dict[str, Any] = {}
self._health_cache_ttl = 60
self._health_cache_lock = threading.Lock()
if use_cache:
self._init_cache()
def _detect_api_type(self, api_key: str) -> str:
"""
Detect if API key is Free or Pro based on suffix.
Free tier keys end with ':fx', Pro keys do not.
Args:
api_key: DeepL API key
Returns:
"free" or "pro"
"""
if api_key.endswith(DEEPL_FREE_SUFFIX):
return "free"
return "pro"
def _get_api_url(self) -> str:
"""
Get correct API URL based on key type.
Note: deep_translator handles this internally, but we log it.
Returns:
API URL for Free or Pro tier
"""
if self._api_type == "free":
return "https://api-free.deepl.com/v2/translate"
return "https://api.deepl.com/v2/translate"
def _init_cache(self):
"""Initialize or get the translation cache."""
from services.translation_service import _translation_cache
self._cache = _translation_cache
def _normalize_language_code(self, lang_code: str) -> str:
"""
Normalize language code for DeepL.
DeepL uses uppercase language codes (e.g., "EN-US", "FR").
Args:
lang_code: Input language code (e.g., "en", "en-US", "EN-us")
Returns:
Normalized language code for DeepL
"""
if not lang_code or lang_code.lower() == "auto":
return ""
lang_upper = lang_code.upper()
if lang_upper in DEEPL_SUPPORTED_LANGUAGES:
return lang_upper
base_lang = lang_upper.split("-")[0]
if base_lang == "EN":
return "EN-US"
elif base_lang == "PT":
return "PT-BR"
elif base_lang in {
"BG",
"CS",
"DA",
"DE",
"EL",
"ES",
"ET",
"FI",
"FR",
"HU",
"ID",
"IT",
"JA",
"KO",
"LT",
"LV",
"NB",
"NL",
"PL",
"RO",
"RU",
"SK",
"SL",
"SV",
"TR",
"UK",
"ZH",
}:
return base_lang
return lang_upper
def _is_language_supported(self, lang_code: str) -> bool:
"""
Check if a language code is supported by DeepL.
Args:
lang_code: Language code to check
Returns:
True if supported, False otherwise
"""
if not lang_code:
return True
normalized = self._normalize_language_code(lang_code)
return normalized in DEEPL_SUPPORTED_LANGUAGES
def _get_translator(self, source_language: str, target_language: str):
"""Get or create a translator instance for the current thread."""
from deep_translator import DeepLTranslator
source_lang = self._normalize_language_code(source_language)
target_lang = self._normalize_language_code(target_language)
key = f"{source_lang}_{target_lang}"
if not hasattr(self._local, "translators"):
self._local.translators = {}
if key not in self._local.translators:
self._local.translators[key] = DeepLTranslator(
api_key=self._api_key,
source=source_lang if source_lang else "auto",
target=target_lang,
)
return self._local.translators[key]
def _make_api_request(
self, text: str, source_language: str, target_language: str
) -> str:
"""
Make API request with error mapping.
Raises:
DeepLProviderError: For any API errors with specific codes
"""
if len(text.encode("utf-8")) > MAX_TEXT_LENGTH:
raise DeepLProviderError(
code=DEEPL_TEXT_TOO_LONG,
message="Texte trop long (max 128KB par requête).",
details={"text_length": len(text), "max_length": MAX_TEXT_LENGTH},
)
if not self._is_language_supported(target_language):
raise DeepLProviderError(
code=DEEPL_UNSUPPORTED_LANGUAGE,
message=f"Langue '{target_language}' non supportée par DeepL.",
details={"unsupported_language": target_language},
)
try:
translator = self._get_translator(source_language, target_language)
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(translator.translate, text)
return future.result(timeout=self.timeout)
except Exception as e:
error_str = str(e).lower()
if (
"quota" in error_str
or "limit" in error_str
or "429" in error_str
or "456" in error_str
):
raise DeepLProviderError(
code=DEEPL_QUOTA_EXCEEDED,
message="Quota DeepL dépassé. Réessayez demain.",
details={"provider": "deepl", "api_type": self._api_type},
)
elif (
"auth" in error_str
or "key" in error_str
or "invalid" in error_str
or "401" in error_str
or "403" in error_str
):
raise DeepLProviderError(
code=DEEPL_INVALID_KEY,
message="Clé API DeepL invalide. Contactez l'administrateur.",
details={"provider": "deepl"},
)
elif "language" in error_str or "not supported" in error_str:
raise DeepLProviderError(
code=DEEPL_UNSUPPORTED_LANGUAGE,
message=f"Langue '{target_language}' non supportée par DeepL.",
details={"unsupported_language": target_language},
)
elif (
isinstance(e, (socket.timeout, TimeoutError, FuturesTimeoutError))
or "timeout" in error_str
):
raise DeepLProviderError(
code=DEEPL_NETWORK_ERROR,
message="Service DeepL indisponible. Réessayez.",
details={"provider": "deepl", "error_type": "timeout"},
)
else:
raise DeepLProviderError(
code=DEEPL_NETWORK_ERROR,
message="Service DeepL indisponible. Réessayez.",
details={"provider": "deepl", "original_error": str(e)[:100]},
)
def get_name(self) -> str:
"""Return provider name."""
return self._provider_name
def is_available(self) -> bool:
"""
Check if DeepL is available (API key configured and API reachable).
Performs a minimal translate call to verify the API is actually reachable.
Uses cached result if available and not expired (TTL 60s).
"""
current_time = time.time()
with self._health_cache_lock:
if "is_available" in self._health_cache:
cached = self._health_cache["is_available"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
available = False
try:
translator = self._get_translator("en", "fr")
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(translator.translate, "a")
future.result(timeout=5)
available = True
except Exception as e:
_log_warning(
"deepl_availability_check_failed",
error=str(e)[:100],
)
with self._health_cache_lock:
self._health_cache["is_available"] = {
"value": available,
"timestamp": current_time,
}
return available
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
"""
Translate a single text string using DeepL.
API Usage Notes:
- DeepL Free tier: 500,000 characters/month
- DeepL Pro: ~€25 per million characters
- 128KB max per request
Optimization: Skips API call if source == target language.
Args:
request: TranslationRequest with text and language info
Returns:
TranslationResponse with translated text
"""
text = request.text
target_language = request.target_language
source_language = request.source_language or "auto"
if not text or not text.strip():
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
)
norm_source = self._normalize_language_code(source_language)
norm_target = self._normalize_language_code(target_language)
if norm_source and norm_source == norm_target:
_log_info(
"deepl_translation_skip",
source_target_lang=target_language,
text_length=len(text),
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
source_language=source_language,
)
if self._use_cache and self._cache:
cached = self._cache.get(
text, target_language, source_language, self._provider_name
)
if cached is not None:
return TranslationResponse(
translated_text=cached,
provider_name=self._provider_name,
from_cache=True,
)
last_error: Optional[DeepLProviderError] = None
retries = 0
while retries <= self.max_retries:
try:
result = self._make_api_request(text, source_language, target_language)
if self._use_cache and self._cache:
self._cache.set(
text,
target_language,
source_language,
self._provider_name,
result,
)
_log_info(
"deepl_translation_success",
chars=len(text),
source_lang=source_language,
target_lang=target_language,
api_type=self._api_type,
retries=retries,
)
return TranslationResponse(
translated_text=result,
provider_name=self._provider_name,
from_cache=False,
)
except DeepLProviderError as e:
last_error = e
if e.code not in _RETRYABLE_ERRORS:
break
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
_log_info(
"deepl_translation_retry",
attempt=retries,
delay_s=round(delay, 2),
error_code=e.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
)
time.sleep(delay)
except Exception as e:
last_error = DeepLProviderError(
code=DEEPL_NETWORK_ERROR,
message="Service DeepL indisponible. Réessayez.",
details={"original_error": str(e)[:100]},
)
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
time.sleep(delay)
if last_error:
_log_error(
"deepl_translation_failed",
error_code=last_error.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
retries=retries,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error=last_error.message,
error_code=last_error.code,
error_details=last_error.details,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error="Unknown error",
error_code=DEEPL_NETWORK_ERROR,
)
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts with optimized batch processing.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
"""
if not requests:
return []
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
"""
Return health status details for the provider.
Performs a lightweight check to verify the provider is operational.
Includes cached result for efficiency.
Returns:
ProviderHealthStatus with availability and latency information
"""
current_time = time.time()
with self._health_cache_lock:
if "health_check" in self._health_cache:
cached = self._health_cache["health_check"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
start_time = time.time()
last_check_iso = datetime.now(timezone.utc).isoformat()
try:
available = self.is_available()
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=available,
latency_ms=round(latency_ms, 2),
error=None if available else "Provider not available",
last_check=last_check_iso,
)
except Exception as e:
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=False,
latency_ms=round(latency_ms, 2),
error=str(e)[:100],
last_check=last_check_iso,
)
with self._health_cache_lock:
self._health_cache["health_check"] = {
"value": status,
"timestamp": current_time,
}
return status
def register_deepl_provider():
"""
Register the DeepL provider in the global registry.
This function should be called during module initialization
to make the provider available through the registry.
"""
from .registry import registry
provider = get_deepl_provider()
if provider:
registry.register("deepl", provider)
return provider
_provider_instance = None
_provider_instance_lock = threading.Lock()
def get_deepl_provider() -> Optional[DeepLTranslationProvider]:
"""Get or create the DeepL provider instance (reads config from env). Thread-safe."""
global _provider_instance
if _provider_instance is None:
with _provider_instance_lock:
if _provider_instance is None:
from .config import ProvidersConfig
if not ProvidersConfig.DEEPL_API_KEY:
return None
_provider_instance = DeepLTranslationProvider(
api_key=ProvidersConfig.DEEPL_API_KEY,
use_cache=True,
timeout=getattr(ProvidersConfig, "DEEPL_TIMEOUT", 30),
max_retries=getattr(ProvidersConfig, "DEEPL_MAX_RETRIES", 3),
retry_delay=getattr(ProvidersConfig, "DEEPL_RETRY_DELAY", 1.0),
)
return _provider_instance
class LegacyDeepLAdapter:
"""
Exposes the new DeepLTranslationProvider via the legacy interface used by
translation_service: .translate(text, target_lang, source_lang) -> str and
.translate_batch(texts, target_lang, source_lang) -> List[str].
Raises TranslationProviderError on failure so the API can return 4xx/502.
"""
def __init__(self):
self._provider = get_deepl_provider()
self.provider_name = "deepl"
def translate(
self, text: str, target_language: str, source_language: str = "auto"
) -> str:
if not self._provider:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
"DEEPL_NOT_CONFIGURED",
"DeepL provider not configured. Set DEEPL_API_KEY.",
None,
)
req = TranslationRequest(
text=text,
target_language=target_language,
source_language=source_language,
)
resp = self._provider.translate_text(req)
if resp.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
resp.error_code or "UNKNOWN",
resp.error or "Translation failed",
resp.error_details,
)
return resp.translated_text
def translate_batch(
self,
texts: List[str],
target_language: str,
source_language: str = "auto",
batch_size: int = 50,
) -> List[str]:
if not self._provider:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
"DEEPL_NOT_CONFIGURED",
"DeepL provider not configured. Set DEEPL_API_KEY.",
None,
)
requests = [
TranslationRequest(
text=t,
target_language=target_language,
source_language=source_language,
)
for t in texts
]
responses = self._provider.translate_batch(requests)
result = []
for r in responses:
if r.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
r.error_code or "UNKNOWN",
r.error or "Translation failed",
r.error_details,
)
result.append(r.translated_text)
return result
def get_legacy_deepl_adapter() -> LegacyDeepLAdapter:
"""Return an adapter so the legacy translation_service can use the new provider."""
return LegacyDeepLAdapter()

345
services/providers/fallback.py Normal file
View File

@@ -0,0 +1,345 @@
"""
Fallback Translation Service - Provider fallback chain implementation.
Provides automatic fallback between translation providers when one fails,
ensuring translation remains available even if individual providers are down.
Features:
- Try providers in order until one succeeds
- Return structured error when all providers fail
- Log failed attempts and successful provider
- Never expose HTTP 500 or document content
"""
from typing import List, Optional, Dict, Any
import time
try:
import structlog
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
"""Log info with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.info(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.info(msg)
def _log_warning(event: str, **kwargs):
"""Log warning with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.warning(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.warning(msg)
def _log_error(event: str, **kwargs):
"""Log error with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.error(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.error(msg)
from .registry import registry
from .schemas import TranslationRequest, TranslationResponse
# Error code for when all providers fail
ALL_PROVIDERS_FAILED = "ALL_PROVIDERS_FAILED"
class AllProvidersFailedError(Exception):
"""
Exception raised when all providers in the fallback chain fail.
This exception is used to signal that no provider could successfully
translate the text, and includes details about which providers were
tried and what errors occurred.
"""
def __init__(
self,
message: str = "Tous les fournisseurs de traduction ont échoué.",
providers_tried: Optional[List[str]] = None,
errors: Optional[List[Dict[str, Any]]] = None,
):
self.code = ALL_PROVIDERS_FAILED
self.message = message
self.providers_tried = providers_tried or []
self.errors = errors or []
super().__init__(message)
def to_dict(self) -> Dict[str, Any]:
"""Convert error to dictionary format for API responses."""
result = {
"error": self.code,
"message": self.message,
"details": {
"providers_tried": self.providers_tried,
"error_count": len(self.errors),
},
}
if self.errors:
# Include last error details (without sensitive info)
last_error = self.errors[-1]
result["details"]["last_error"] = {
"provider": last_error.get("provider"),
"error_code": last_error.get("error_code"),
"message": last_error.get("message", "")[:200], # Truncate
}
return result
def translate_with_fallback(
request: TranslationRequest,
provider_names: List[str],
skip_unavailable: bool = True,
) -> TranslationResponse:
"""
Translate text using a fallback chain of providers.
Iterates through the list of provider names in order, attempting to
translate with each one. Returns the first successful translation.
If all providers fail, raises AllProvidersFailedError.
Args:
request: TranslationRequest with text and language info
provider_names: Ordered list of provider names to try
skip_unavailable: If True, skip providers that are not available
(health check fails). If False, try anyway.
Returns:
TranslationResponse with translated text and provider_name set
to the successful provider.
Raises:
AllProvidersFailedError: When all providers in the chain fail
Example:
>>> request = TranslationRequest(text="Hello", target_language="fr")
>>> response = translate_with_fallback(
... request, ["google", "deepl", "openai"]
... )
>>> print(response.translated_text) # "Bonjour"
>>> print(response.provider_name) # "deepl" (first that succeeded)
"""
if not provider_names:
raise AllProvidersFailedError(
message="Aucun fournisseur configuré dans la chaîne de fallback.",
providers_tried=[],
)
providers_tried: List[str] = []
errors: List[Dict[str, Any]] = []
_log_info(
"fallback_translation_started",
providers=provider_names,
source_lang=request.source_language,
target_lang=request.target_language,
text_length=len(request.text),
)
for provider_name in provider_names:
# Get provider from registry
provider = registry.get(provider_name)
if provider is None:
_log_warning(
"fallback_provider_not_registered",
provider=provider_name,
)
errors.append(
{
"provider": provider_name,
"error_code": "PROVIDER_NOT_REGISTERED",
"message": f"Provider '{provider_name}' not registered",
}
)
providers_tried.append(provider_name)
continue
# Check availability if requested
if skip_unavailable and not provider.is_available():
_log_info(
"fallback_provider_unavailable",
provider=provider_name,
)
errors.append(
{
"provider": provider_name,
"error_code": "PROVIDER_UNAVAILABLE",
"message": f"Provider '{provider_name}' is not available",
}
)
providers_tried.append(provider_name)
continue
# Try to translate
start_time = time.time()
try:
response = provider.translate_text(request)
latency_ms = (time.time() - start_time) * 1000
# Check if translation succeeded
if response.error is None:
# Success!
_log_info(
"fallback_translation_success",
provider=provider_name,
latency_ms=round(latency_ms, 2),
attempts=len(providers_tried) + 1,
text_length=len(request.text),
source_lang=request.source_language,
target_lang=request.target_language,
)
# Ensure provider_name is set
if not response.provider_name:
response.provider_name = provider_name
return response
else:
# Provider returned an error
_log_warning(
"fallback_provider_error",
provider=provider_name,
error_code=response.error_code,
error_message=response.error[:200], # Truncate
)
errors.append(
{
"provider": provider_name,
"error_code": response.error_code,
"message": response.error,
}
)
providers_tried.append(provider_name)
except Exception as e:
# Provider raised an exception
latency_ms = (time.time() - start_time) * 1000
error_str = str(e)
_log_error(
"fallback_provider_exception",
provider=provider_name,
error_type=type(e).__name__,
latency_ms=round(latency_ms, 2),
)
errors.append(
{
"provider": provider_name,
"error_code": "PROVIDER_EXCEPTION",
"message": error_str[:200], # Truncate
}
)
providers_tried.append(provider_name)
# All providers failed
_log_error(
"fallback_all_providers_failed",
providers_tried=providers_tried,
error_count=len(errors),
text_length=len(request.text),
source_lang=request.source_language,
target_lang=request.target_language,
)
raise AllProvidersFailedError(
message="Tous les fournisseurs de traduction ont échoué. Veuillez réessayer plus tard.",
providers_tried=providers_tried,
errors=errors,
)
def translate_with_fallback_by_mode(
request: TranslationRequest,
mode: str = "auto",
) -> TranslationResponse:
"""
Translate text using the fallback chain for a specific mode.
Args:
request: TranslationRequest with text and language info
mode: "classic" for Classic providers, "llm" for LLM providers,
"auto" for general fallback chain
Returns:
TranslationResponse with translated text
Raises:
AllProvidersFailedError: When all providers fail
"""
from .config import ProvidersConfig
provider_names = ProvidersConfig.get_fallback_chain(mode)
if not provider_names:
raise AllProvidersFailedError(
message=f"Aucune chaîne de fallback configurée pour le mode '{mode}'.",
providers_tried=[],
)
return translate_with_fallback(request, provider_names)
class LegacyFallbackAdapter:
"""
Exposes the fallback chain via the legacy interface used by translation_service:
.translate(text, target_lang, source_lang) -> str and
.translate_batch(texts, target_lang, source_lang) -> List[str].
Raises AllProvidersFailedError when all providers fail (API returns 502).
"""
def __init__(self, mode: str = "classic"):
"""
Args:
mode: "classic" (Google → DeepL) or "llm" (Ollama → OpenAI)
"""
self._mode = mode.lower()
self.provider_name = f"fallback_{self._mode}"
self._last_provider_used: Optional[str] = None
def translate(
self, text: str, target_language: str, source_language: str = "auto"
) -> str:
req = TranslationRequest(
text=text,
target_language=target_language,
source_language=source_language,
)
response = translate_with_fallback_by_mode(req, self._mode)
self._last_provider_used = response.provider_name or self._last_provider_used
return response.translated_text
def translate_batch(
self,
texts: List[str],
target_language: str,
source_language: str = "auto",
batch_size: int = 50,
) -> List[str]:
results: List[str] = []
for t in texts:
req = TranslationRequest(
text=t,
target_language=target_language,
source_language=source_language,
)
response = translate_with_fallback_by_mode(req, self._mode)
self._last_provider_used = response.provider_name or self._last_provider_used
results.append(response.translated_text)
return results

534
services/providers/google_provider.py Normal file
View File

@@ -0,0 +1,534 @@
"""
Google Translate Provider - Production-ready implementation.
Extends TranslationProvider base class with robust error handling,
retry logic, and health monitoring.
Features:
- Specific error codes for all Google API errors
- Retry logic with exponential backoff for transient errors
- Timeout configuration
- Health check with caching
- Structlog-compatible logging (no document content in logs)
"""
import os
import socket
import threading
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
logger = structlog.get_logger(__name__)
except ImportError:
import logging
logger = logging.getLogger(__name__)
from .base import TranslationProvider
from .schemas import (
BatchTranslationRequest,
BatchTranslationResponse,
ProviderHealthStatus,
TranslationRequest,
TranslationResponse,
)
GOOGLE_QUOTA_EXCEEDED = "GOOGLE_QUOTA_EXCEEDED"
GOOGLE_INVALID_KEY = "GOOGLE_INVALID_KEY"
GOOGLE_NETWORK_ERROR = "GOOGLE_NETWORK_ERROR"
GOOGLE_UNSUPPORTED_LANGUAGE = "GOOGLE_UNSUPPORTED_LANGUAGE"
GOOGLE_TEXT_TOO_LONG = "GOOGLE_TEXT_TOO_LONG"
_RETRYABLE_ERRORS = {GOOGLE_NETWORK_ERROR, GOOGLE_QUOTA_EXCEEDED}
class GoogleProviderError(Exception):
"""Exception raised for Google Translate API errors."""
def __init__(
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
):
self.code = code
self.message = message
self.details = details or {}
super().__init__(message)
def to_dict(self) -> Dict[str, Any]:
"""Convert error to dictionary format."""
result = {
"error": self.code,
"message": self.message,
}
if self.details:
result["details"] = self.details
return result
class GoogleTranslationProvider(TranslationProvider):
"""
Google Translate implementation using deep_translator library.
Features:
- Thread-safe translator instances per thread
- Caching support (uses global cache from translation_service)
- Batch translation with optimized processing
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Configurable timeout
- Health check with result caching
"""
def __init__(
self,
use_cache: bool = True,
timeout: int = 30,
max_retries: int = 3,
retry_delay: float = 1.0,
):
"""
Initialize Google Translate provider.
Args:
use_cache: Whether to use translation caching (default: True)
timeout: Request timeout in seconds (default: 30)
max_retries: Maximum retry attempts for transient errors (default: 3)
retry_delay: Initial retry delay in seconds (default: 1.0)
"""
self._local = threading.local()
self._use_cache = use_cache
self._provider_name = "google"
self._cache = None
self.timeout = timeout
self.max_retries = max_retries
self.retry_delay = retry_delay
self._health_cache: Dict[str, Any] = {}
self._health_cache_ttl = 60
self._health_cache_lock = threading.Lock()
if use_cache:
self._init_cache()
def _init_cache(self):
"""Initialize or get the translation cache."""
from services.translation_service import _translation_cache
self._cache = _translation_cache
def _get_translator(self, source_language: str, target_language: str):
"""Get or create a translator instance for the current thread."""
from deep_translator import GoogleTranslator
key = f"{source_language}_{target_language}"
if not hasattr(self._local, "translators"):
self._local.translators = {}
if key not in self._local.translators:
self._local.translators[key] = GoogleTranslator(
source=source_language, target=target_language
)
return self._local.translators[key]
def _make_api_request(
self, text: str, source_language: str, target_language: str
) -> str:
"""
Make API request with error mapping.
Raises:
GoogleProviderError: For any API errors with specific codes
"""
if len(text) > 5000:
raise GoogleProviderError(
code=GOOGLE_TEXT_TOO_LONG,
message="Texte trop long (max 5000 caractères par requête).",
details={"text_length": len(text), "max_length": 5000},
)
try:
translator = self._get_translator(source_language, target_language)
# Apply timeout via executor (deep_translator has no timeout parameter)
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(translator.translate, text)
return future.result(timeout=self.timeout)
except Exception as e:
error_str = str(e).lower()
if "quota" in error_str or "limit" in error_str or "429" in error_str:
raise GoogleProviderError(
code=GOOGLE_QUOTA_EXCEEDED,
message="Quota Google Translate dépassé. Réessayez demain.",
details={"provider": "google"},
)
elif "api" in error_str and (
"key" in error_str
or "invalid" in error_str
or "401" in error_str
or "403" in error_str
):
raise GoogleProviderError(
code=GOOGLE_INVALID_KEY,
message="Clé API Google invalide. Contactez l'administrateur.",
details={"provider": "google"},
)
elif "language" in error_str or "not supported" in error_str:
raise GoogleProviderError(
code=GOOGLE_UNSUPPORTED_LANGUAGE,
message=f"Langue '{target_language}' non supportée par Google.",
details={"unsupported_language": target_language},
)
elif (
isinstance(e, (socket.timeout, TimeoutError, FuturesTimeoutError))
or "timeout" in error_str
):
raise GoogleProviderError(
code=GOOGLE_NETWORK_ERROR,
message="Service Google Translate indisponible. Réessayez.",
details={"provider": "google", "error_type": "timeout"},
)
else:
raise GoogleProviderError(
code=GOOGLE_NETWORK_ERROR,
message="Service Google Translate indisponible. Réessayez.",
details={"provider": "google", "original_error": str(e)[:100]},
)
def get_name(self) -> str:
"""Return provider name."""
return self._provider_name
def is_available(self) -> bool:
"""
Check if Google Translate is available.
Uses cached result if available and not expired.
"""
current_time = time.time()
with self._health_cache_lock:
if "is_available" in self._health_cache:
cached = self._health_cache["is_available"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
try:
translator = self._get_translator("auto", "en")
available = translator is not None
except Exception as e:
logger.warning(
"google_availability_check_failed",
error=str(e)[:100],
)
available = False
with self._health_cache_lock:
self._health_cache["is_available"] = {
"value": available,
"timestamp": current_time,
}
return available
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
"""
Translate a single text string using Google Translate.
API Usage Notes:
- Google Translate free tier: 500,000 characters/month
- 5,000 characters max per request
- Cost: ~$20 per million characters (paid tier)
Optimization: Skips API call if source == target language.
Args:
request: TranslationRequest with text and language info
Returns:
TranslationResponse with translated text
"""
text = request.text
target_language = request.target_language
source_language = request.source_language or "auto"
if not text or not text.strip():
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
)
# Optimization: Skip if source and target are the same
if source_language != "auto" and source_language == target_language:
logger.info(
"google_translation_skip",
source_target_lang=target_language,
text_length=len(text),
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
source_language=source_language,
)
if self._use_cache and self._cache:
cached = self._cache.get(
text, target_language, source_language, self._provider_name
)
if cached is not None:
return TranslationResponse(
translated_text=cached,
provider_name=self._provider_name,
from_cache=True,
)
last_error: Optional[GoogleProviderError] = None
retries = 0
while retries <= self.max_retries:
try:
result = self._make_api_request(text, source_language, target_language)
if self._use_cache and self._cache:
self._cache.set(
text,
target_language,
source_language,
self._provider_name,
result,
)
# Log usage metrics (character count, API call)
logger.info(
"google_translation_success",
chars=len(text),
source_lang=source_language,
target_lang=target_language,
retries=retries,
)
return TranslationResponse(
translated_text=result,
provider_name=self._provider_name,
from_cache=False,
)
except GoogleProviderError as e:
last_error = e
if e.code not in _RETRYABLE_ERRORS:
break
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
logger.info(
"google_translation_retry",
attempt=retries,
delay_s=round(delay, 2),
error_code=e.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
)
time.sleep(delay)
except Exception as e:
last_error = GoogleProviderError(
code=GOOGLE_NETWORK_ERROR,
message="Service Google Translate indisponible. Réessayez.",
details={"original_error": str(e)[:100]},
)
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
time.sleep(delay)
if last_error:
logger.error(
"google_translation_failed",
error_code=last_error.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
retries=retries,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error=last_error.message,
error_code=last_error.code,
error_details=last_error.details,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error="Unknown error",
error_code=GOOGLE_NETWORK_ERROR,
)
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts with optimized batch processing.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
"""
if not requests:
return []
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
"""
Return health status details for the provider.
Performs a lightweight check to verify the provider is operational.
Includes cached result for efficiency.
Returns:
ProviderHealthStatus with availability and latency information
"""
current_time = time.time()
with self._health_cache_lock:
if "health_check" in self._health_cache:
cached = self._health_cache["health_check"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
start_time = time.time()
last_check_iso = datetime.now(timezone.utc).isoformat()
try:
available = self.is_available()
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=available,
latency_ms=round(latency_ms, 2),
error=None if available else "Provider not available",
last_check=last_check_iso,
)
except Exception as e:
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=False,
latency_ms=round(latency_ms, 2),
error=str(e)[:100],
last_check=last_check_iso,
)
with self._health_cache_lock:
self._health_cache["health_check"] = {
"value": status,
"timestamp": current_time,
}
return status
def register_google_provider():
"""
Register the Google provider in the global registry.
This function should be called during module initialization
to make the provider available through the registry.
"""
from .registry import registry
provider = get_google_provider()
registry.register("google", provider)
return provider
_provider_instance = None
def get_google_provider() -> GoogleTranslationProvider:
"""Get or create the Google provider instance (reads config from env)."""
global _provider_instance
if _provider_instance is None:
from .config import ProvidersConfig
_provider_instance = GoogleTranslationProvider(
use_cache=True,
timeout=ProvidersConfig.GOOGLE_TRANSLATE_TIMEOUT,
max_retries=ProvidersConfig.GOOGLE_TRANSLATE_MAX_RETRIES,
retry_delay=ProvidersConfig.GOOGLE_TRANSLATE_RETRY_DELAY,
)
return _provider_instance
class LegacyGoogleAdapter:
"""
Exposes the new GoogleTranslationProvider via the legacy interface used by
translation_service: .translate(text, target_lang, source_lang) -> str and
.translate_batch(texts, target_lang, source_lang) -> List[str].
Raises TranslationProviderError on failure so the API can return 4xx/502.
"""
def __init__(self):
self._provider = get_google_provider()
self.provider_name = "google"
def translate(
self, text: str, target_language: str, source_language: str = "auto"
) -> str:
req = TranslationRequest(
text=text,
target_language=target_language,
source_language=source_language,
)
resp = self._provider.translate_text(req)
if resp.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
resp.error_code or "UNKNOWN",
resp.error or "Translation failed",
resp.error_details,
)
return resp.translated_text
def translate_batch(
self,
texts: List[str],
target_language: str,
source_language: str = "auto",
batch_size: int = 50,
) -> List[str]:
requests = [
TranslationRequest(
text=t,
target_language=target_language,
source_language=source_language,
)
for t in texts
]
responses = self._provider.translate_batch(requests)
result = []
for r in responses:
if r.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(
r.error_code or "UNKNOWN",
r.error or "Translation failed",
r.error_details,
)
result.append(r.translated_text)
return result
def get_legacy_google_adapter() -> LegacyGoogleAdapter:
"""Return an adapter so the legacy translation_service can use the new provider."""
return LegacyGoogleAdapter()

605
services/providers/ollama_provider.py Normal file
View File

@@ -0,0 +1,605 @@
"""
Ollama Provider - Local LLM translation provider.
Extends TranslationProvider base class with robust error handling,
retry logic, and health monitoring for local Ollama instances.
Features:
- Local LLM translation via Ollama REST API
- Custom system prompt support
- Specific error codes for all Ollama API errors
- Retry logic with exponential backoff for transient errors
- Timeout configuration (longer for LLM)
- Health check with caching
- Structlog-compatible logging (no document content in logs)
"""
import socket
import threading
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from urllib.parse import urljoin
try:
import structlog
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
"""Log info with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.info(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.info(msg)
def _log_warning(event: str, **kwargs):
"""Log warning with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.warning(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.warning(msg)
def _log_error(event: str, **kwargs):
"""Log error with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.error(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.error(msg)
import requests
from requests.exceptions import Timeout, ConnectionError as RequestsConnectionError
from .base import TranslationProvider
from .schemas import (
ProviderHealthStatus,
TranslationRequest,
TranslationResponse,
)
OLLAMA_UNAVAILABLE = "OLLAMA_UNAVAILABLE"
OLLAMA_MODEL_NOT_FOUND = "OLLAMA_MODEL_NOT_FOUND"
OLLAMA_TIMEOUT = "OLLAMA_TIMEOUT"
OLLAMA_GENERATION_ERROR = "OLLAMA_GENERATION_ERROR"
OLLAMA_CONTEXT_TOO_LONG = "OLLAMA_CONTEXT_TOO_LONG"
_RETRYABLE_ERRORS = {OLLAMA_UNAVAILABLE, OLLAMA_TIMEOUT}
class OllamaProviderError(Exception):
"""Exception raised for Ollama API errors."""
def __init__(
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
):
self.code = code
self.message = message
self.details = details or {}
super().__init__(message)
def to_dict(self) -> Dict[str, Any]:
"""Convert error to dictionary format."""
result = {
"error": self.code,
"message": self.message,
}
if self.details:
result["details"] = self.details
return result
DEFAULT_TRANSLATION_PROMPT = """You are a professional translator. Translate the following text from {source_lang} to {target_lang}.
Rules:
- Translate ONLY the text, do not add explanations or notes
- Preserve the original formatting, line breaks, and structure
- Maintain the original tone and style
- For technical terms, use the standard translation in the target language
- If the text contains proper nouns or brand names, keep them unchanged unless there's a well-known translation"""
def _build_system_prompt(
source_lang: str, target_lang: str, custom_prompt: Optional[str] = None
) -> str:
"""Build system prompt for translation."""
if custom_prompt:
return custom_prompt
return DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang, target_lang=target_lang
)
def _get_language_name(code: str) -> str:
"""Convert language code to full name for better LLM understanding."""
language_names = {
"en": "English",
"fr": "French",
"es": "Spanish",
"de": "German",
"it": "Italian",
"pt": "Portuguese",
"nl": "Dutch",
"ru": "Russian",
"zh": "Chinese",
"ja": "Japanese",
"ko": "Korean",
"ar": "Arabic",
"hi": "Hindi",
"tr": "Turkish",
"pl": "Polish",
"vi": "Vietnamese",
"th": "Thai",
"id": "Indonesian",
"ms": "Malay",
"uk": "Ukrainian",
"cs": "Czech",
"sv": "Swedish",
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"el": "Greek",
"he": "Hebrew",
"ro": "Romanian",
"hu": "Hungarian",
"bg": "Bulgarian",
"sk": "Slovak",
"hr": "Croatian",
"sl": "Slovenian",
"lt": "Lithuanian",
"lv": "Latvian",
"et": "Estonian",
}
base_code = code.split("-")[0].lower()
return language_names.get(base_code, code)
class OllamaTranslationProvider(TranslationProvider):
"""
Ollama LLM implementation for local translation.
Features:
- Uses Ollama REST API (/api/chat endpoint)
- Custom system prompt support for translation context
- Thread-safe HTTP client
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Configurable timeout (default 120s for LLM)
- Health check with result caching
"""
def __init__(
self,
base_url: str = "http://localhost:11434",
model: Optional[str] = None,
timeout: int = 120,
max_retries: int = 2,
retry_delay: float = 2.0,
):
"""
Initialize Ollama provider.
Args:
base_url: Ollama API base URL (default: http://localhost:11434)
model: Model name (e.g. llama3, mistral). If None, uses OLLAMA_MODEL from config.
timeout: Request timeout in seconds (default: 120 for LLM)
max_retries: Maximum retry attempts for transient errors (default: 2)
retry_delay: Initial retry delay in seconds (default: 2.0)
"""
if model is None:
from .config import ProvidersConfig
model = ProvidersConfig.OLLAMA_MODEL
self._base_url = base_url.rstrip("/")
self._model = model
self._provider_name = "ollama"
self.timeout = timeout
self.max_retries = max_retries
self.retry_delay = retry_delay
self._health_cache: Dict[str, Any] = {}
self._health_cache_ttl = 60
self._health_cache_lock = threading.Lock()
self._available_models: Optional[List[str]] = None
self._models_cache_time: float = 0
self._models_cache_ttl = 300
def _fetch_available_models(self) -> List[str]:
"""Fetch list of available (pulled) models from Ollama."""
current_time = time.time()
if (
self._available_models is not None
and current_time - self._models_cache_time < self._models_cache_ttl
):
return self._available_models
try:
response = requests.get(f"{self._base_url}/api/tags", timeout=10)
if response.status_code == 200:
data = response.json()
models = [m.get("name", "") for m in data.get("models", [])]
self._available_models = models
self._models_cache_time = current_time
return models
except Exception as e:
_log_warning("ollama_models_fetch_failed", error=str(e)[:100])
return []
def _check_model_available(self, model: str) -> bool:
"""Check if a specific model is available (pulled)."""
models = self._fetch_available_models()
return any(m.startswith(model) or model in m for m in models)
def _make_api_request(self, text: str, system_prompt: str) -> str:
"""
Make API request to Ollama.
Raises:
OllamaProviderError: For any API errors with specific codes
"""
if not text or not text.strip():
return text
if len(text) > 128000:
raise OllamaProviderError(
code=OLLAMA_CONTEXT_TOO_LONG,
message="Texte trop long pour le modèle (max ~128K caractères).",
details={"text_length": len(text), "max_chars": 128000},
)
if not self._check_model_available(self._model):
raise OllamaProviderError(
code=OLLAMA_MODEL_NOT_FOUND,
message=f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}",
details={"model": self._model, "provider": "ollama"},
)
payload = {
"model": self._model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": text},
],
"stream": False,
"options": {"temperature": 0.3},
}
try:
response = requests.post(
f"{self._base_url}/api/chat",
json=payload,
timeout=self.timeout,
)
if response.status_code == 404:
raise OllamaProviderError(
code=OLLAMA_MODEL_NOT_FOUND,
message=f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}",
details={"model": self._model, "status_code": 404},
)
if response.status_code != 200:
error_text = response.text[:200] if response.text else "Unknown error"
raise OllamaProviderError(
code=OLLAMA_GENERATION_ERROR,
message=f"Erreur de génération Ollama: {error_text}",
details={"status_code": response.status_code, "model": self._model},
)
data = response.json()
message = data.get("message", {})
content = message.get("content", "")
if not content:
raise OllamaProviderError(
code=OLLAMA_GENERATION_ERROR,
message="Erreur de génération Ollama: réponse vide",
details={"model": self._model, "response": str(data)[:200]},
)
return content.strip()
except Timeout:
raise OllamaProviderError(
code=OLLAMA_TIMEOUT,
message="Délai d'attente Ollama dépassé. Réessayez avec un texte plus court.",
details={"provider": "ollama", "timeout_seconds": self.timeout},
)
except RequestsConnectionError:
raise OllamaProviderError(
code=OLLAMA_UNAVAILABLE,
message="Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution.",
details={"provider": "ollama", "base_url": self._base_url},
)
except OllamaProviderError:
raise
except Exception as e:
error_str = str(e).lower()
if "connection" in error_str or "refused" in error_str:
raise OllamaProviderError(
code=OLLAMA_UNAVAILABLE,
message="Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution.",
details={"provider": "ollama", "base_url": self._base_url},
)
raise OllamaProviderError(
code=OLLAMA_GENERATION_ERROR,
message=f"Erreur de génération Ollama: {str(e)[:100]}",
details={"provider": "ollama", "original_error": str(e)[:100]},
)
def get_name(self) -> str:
"""Return provider name."""
return self._provider_name
def is_available(self) -> bool:
"""
Check if Ollama is available.
Uses cached result if available and not expired.
"""
current_time = time.time()
with self._health_cache_lock:
if "is_available" in self._health_cache:
cached = self._health_cache["is_available"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
try:
response = requests.get(f"{self._base_url}/api/tags", timeout=5)
available = response.status_code == 200
except Exception as e:
_log_warning("ollama_availability_check_failed", error=str(e)[:100])
available = False
with self._health_cache_lock:
self._health_cache["is_available"] = {
"value": available,
"timestamp": current_time,
}
return available
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
"""
Translate a single text string using Ollama LLM.
Supports custom system prompt via request.metadata["custom_prompt"].
Args:
request: TranslationRequest with text and language info
Returns:
TranslationResponse with translated text
"""
text = request.text
target_language = request.target_language
source_language = request.source_language or "auto"
if not text or not text.strip():
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
)
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = None
if request.metadata:
custom_prompt = request.metadata.get("custom_prompt")
system_prompt = _build_system_prompt(
source_lang_name, target_lang_name, custom_prompt
)
last_error: Optional[OllamaProviderError] = None
retries = 0
while retries <= self.max_retries:
try:
start_time = time.time()
result = self._make_api_request(text, system_prompt)
latency = time.time() - start_time
_log_info(
"ollama_translation_success",
chars=len(text),
source_lang=source_language,
target_lang=target_language,
model=self._model,
latency_ms=round(latency * 1000, 2),
retries=retries,
)
return TranslationResponse(
translated_text=result,
provider_name=self._provider_name,
from_cache=False,
source_language=source_language,
)
except OllamaProviderError as e:
last_error = e
if e.code not in _RETRYABLE_ERRORS:
break
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
_log_info(
"ollama_translation_retry",
attempt=retries,
delay_s=round(delay, 2),
error_code=e.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
)
time.sleep(delay)
except Exception as e:
last_error = OllamaProviderError(
code=OLLAMA_GENERATION_ERROR,
message=f"Erreur de génération Ollama: {str(e)[:100]}",
details={"original_error": str(e)[:100]},
)
retries += 1
if retries <= self.max_retries:
delay = self.retry_delay * (2 ** (retries - 1))
time.sleep(delay)
if last_error:
_log_error(
"ollama_translation_failed",
error_code=last_error.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
retries=retries,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error=last_error.message,
error_code=last_error.code,
error_details=last_error.details,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error="Unknown error",
error_code=OLLAMA_GENERATION_ERROR,
)
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
"""
if not requests:
return []
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
"""
Return health status details for the provider.
Includes cached result for efficiency.
Returns:
ProviderHealthStatus with availability and latency information
"""
current_time = time.time()
with self._health_cache_lock:
if "health_check" in self._health_cache:
cached = self._health_cache["health_check"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
start_time = time.time()
last_check_iso = datetime.now(timezone.utc).isoformat()
try:
models = self._fetch_available_models()
model_available = self._check_model_available(self._model)
available = len(models) > 0 and model_available
latency_ms = (time.time() - start_time) * 1000
error_msg = None
if not available and len(models) == 0:
error_msg = "Service Ollama indisponible. Vérifiez que Ollama est en cours d'exécution."
elif not model_available:
error_msg = f"Modèle '{self._model}' non trouvé. Exécutez: ollama pull {self._model}"
status = ProviderHealthStatus(
name=self._provider_name,
available=available,
latency_ms=round(latency_ms, 2),
error=error_msg,
last_check=last_check_iso,
model=self._model,
model_available=model_available,
)
except Exception as e:
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=False,
latency_ms=round(latency_ms, 2),
error=str(e)[:100],
last_check=last_check_iso,
model=self._model,
model_available=None,
)
with self._health_cache_lock:
self._health_cache["health_check"] = {
"value": status,
"timestamp": current_time,
}
return status
def register_ollama_provider():
"""
Register the Ollama provider in the global registry.
This function should be called during module initialization
to make the provider available through the registry.
"""
from .registry import registry
provider = get_ollama_provider()
registry.register("ollama", provider)
return provider
_provider_instance: Optional[OllamaTranslationProvider] = None
_provider_lock = threading.Lock()
def get_ollama_provider() -> OllamaTranslationProvider:
"""Get or create the Ollama provider instance (reads config from env)."""
global _provider_instance
if _provider_instance is None:
with _provider_lock:
if _provider_instance is None:
from .config import ProvidersConfig
_provider_instance = OllamaTranslationProvider(
base_url=ProvidersConfig.OLLAMA_BASE_URL,
model=ProvidersConfig.OLLAMA_MODEL,
timeout=ProvidersConfig.OLLAMA_TIMEOUT,
max_retries=ProvidersConfig.OLLAMA_MAX_RETRIES,
retry_delay=ProvidersConfig.OLLAMA_RETRY_DELAY,
)
return _provider_instance

670
services/providers/openai_provider.py Normal file
View File

@@ -0,0 +1,670 @@
"""
OpenAI Provider - Cloud LLM translation provider.
Extends TranslationProvider base class with robust error handling,
retry logic, and health monitoring for OpenAI API.
Features:
- Cloud LLM translation via OpenAI Chat Completions API
- Custom system prompt support
- Specific error codes for all OpenAI API errors
- Retry logic with exponential backoff for transient errors
- Timeout configuration (faster than local Ollama)
- Health check with caching
- Structlog-compatible logging (no document content in logs)
"""
import threading
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
"""Log info with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.info(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.info(msg)
def _log_warning(event: str, **kwargs):
"""Log warning with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.warning(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.warning(msg)
def _log_error(event: str, **kwargs):
"""Log error with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.error(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.error(msg)
import requests
from requests.exceptions import Timeout, ConnectionError as RequestsConnectionError
from .base import TranslationProvider
from .schemas import (
ProviderHealthStatus,
TranslationRequest,
TranslationResponse,
)
# Error codes
OPENAI_RATE_LIMITED = "OPENAI_RATE_LIMITED"
OPENAI_INVALID_KEY = "OPENAI_INVALID_KEY"
OPENAI_QUOTA_EXCEEDED = "OPENAI_QUOTA_EXCEEDED"
OPENAI_TIMEOUT = "OPENAI_TIMEOUT"
OPENAI_SERVICE_ERROR = "OPENAI_SERVICE_ERROR"
OPENAI_CONTEXT_TOO_LONG = "OPENAI_CONTEXT_TOO_LONG"
_RETRYABLE_ERRORS = {OPENAI_RATE_LIMITED, OPENAI_TIMEOUT, OPENAI_SERVICE_ERROR}
class OpenAIProviderError(Exception):
"""Exception raised for OpenAI API errors."""
def __init__(
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
):
self.code = code
self.message = message
self.details = details or {}
super().__init__(message)
def to_dict(self) -> Dict[str, Any]:
"""Convert error to dictionary format."""
result = {
"error": self.code,
"message": self.message,
}
if self.details:
result["details"] = self.details
return result
DEFAULT_TRANSLATION_PROMPT = """You are a professional translator. Translate the following text from {source_lang} to {target_lang}.
Rules:
- Translate ONLY the text, do not add explanations or notes
- Preserve the original formatting, line breaks, and structure
- Maintain the original tone and style
- For technical terms, use the standard translation in the target language
- If the text contains proper nouns or brand names, keep them unchanged unless there's a well-known translation"""
def _build_system_prompt(
source_lang: str, target_lang: str, custom_prompt: Optional[str] = None
) -> str:
"""Build system prompt for translation."""
if custom_prompt:
return custom_prompt
return DEFAULT_TRANSLATION_PROMPT.format(
source_lang=source_lang, target_lang=target_lang
)
def _get_language_name(code: str) -> str:
"""Convert language code to full name for better LLM understanding."""
language_names = {
"en": "English",
"fr": "French",
"es": "Spanish",
"de": "German",
"it": "Italian",
"pt": "Portuguese",
"nl": "Dutch",
"ru": "Russian",
"zh": "Chinese",
"ja": "Japanese",
"ko": "Korean",
"ar": "Arabic",
"hi": "Hindi",
"tr": "Turkish",
"pl": "Polish",
"vi": "Vietnamese",
"th": "Thai",
"id": "Indonesian",
"ms": "Malay",
"uk": "Ukrainian",
"cs": "Czech",
"sv": "Swedish",
"da": "Danish",
"fi": "Finnish",
"no": "Norwegian",
"el": "Greek",
"he": "Hebrew",
"ro": "Romanian",
"hu": "Hungarian",
"bg": "Bulgarian",
"sk": "Slovak",
"hr": "Croatian",
"sl": "Slovenian",
"lt": "Lithuanian",
"lv": "Latvian",
"et": "Estonian",
}
base_code = code.split("-")[0].lower()
return language_names.get(base_code, code)
class OpenAITranslationProvider(TranslationProvider):
"""
OpenAI LLM implementation for cloud translation.
Features:
- Uses OpenAI Chat Completions API
- Custom system prompt support for translation context
- Thread-safe HTTP client
- Robust error handling with specific error codes
- Retry logic with exponential backoff
- Configurable timeout (default 60s for cloud API)
- Health check with result caching
"""
def __init__(
self,
api_key: str,
model: str = "gpt-4o-mini",
timeout: int = 60,
max_retries: int = 3,
retry_delay: float = 1.0,
base_url: str = "https://api.openai.com/v1",
health_check_timeout: int = 5,
):
"""
Initialize OpenAI provider.
Args:
api_key: OpenAI API key
model: Model name to use (default: gpt-4o-mini)
timeout: Request timeout in seconds (default: 60)
max_retries: Maximum retry attempts for transient errors (default: 3)
retry_delay: Initial retry delay in seconds (default: 1.0)
base_url: OpenAI API base URL (default: https://api.openai.com/v1)
health_check_timeout: Timeout for health check requests in seconds (default: 5)
"""
if not api_key or not api_key.strip():
raise ValueError("OpenAI API key cannot be empty")
self._api_key = api_key
self._model = model
self._base_url = base_url.rstrip("/")
self._provider_name = "openai"
self._timeout = timeout
self._max_retries = max_retries
self._retry_delay = retry_delay
self._health_check_timeout = health_check_timeout
self._health_cache: Dict[str, Any] = {}
self._health_cache_ttl = 60
self._health_cache_lock = threading.Lock()
def _make_api_request(self, text: str, system_prompt: str) -> tuple:
"""
Make API request to OpenAI.
Returns:
Tuple of (translated_content, usage_dict). usage_dict may be empty.
Raises:
OpenAIProviderError: For any API errors with specific codes
"""
if not text or not text.strip():
return text, {}
# Check text length (rough estimate: 1 token ~= 4 chars)
if len(text) > 16000: # ~4000 tokens
raise OpenAIProviderError(
code=OPENAI_CONTEXT_TOO_LONG,
message="Texte trop long pour le modèle (max ~4000 tokens).",
details={"text_length": len(text), "max_tokens": 4000},
)
url = f"{self._base_url}/chat/completions"
headers = {
"Authorization": f"Bearer {self._api_key}",
"Content-Type": "application/json",
}
payload = {
"model": self._model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": text},
],
"temperature": 0.3,
"max_tokens": 4096,
}
try:
response = requests.post(
url,
headers=headers,
json=payload,
timeout=self._timeout,
)
# Handle specific HTTP status codes
if response.status_code == 401:
raise OpenAIProviderError(
code=OPENAI_INVALID_KEY,
message="Clé API OpenAI invalide. Vérifiez votre configuration.",
details={"status_code": 401},
)
if response.status_code == 429:
try:
error_data = response.json().get("error", {}) or {}
except Exception:
error_data = {}
error_code = error_data.get("code", "")
# Check for rate limit vs quota exceeded
if error_code == "insufficient_quota":
raise OpenAIProviderError(
code=OPENAI_QUOTA_EXCEEDED,
message="Quota OpenAI épuisé. Vérifiez votre facturation.",
details={"status_code": 429, "error_code": error_code},
)
else:
# Rate limit
retry_after = response.headers.get("retry-after", "20")
raise OpenAIProviderError(
code=OPENAI_RATE_LIMITED,
message=f"Limite de requêtes OpenAI atteinte. Réessayez dans {retry_after}s.",
details={
"status_code": 429,
"retry_after_seconds": int(retry_after)
if retry_after.isdigit()
else 20,
},
)
if response.status_code == 400:
try:
error_data = response.json().get("error", {}) or {}
except Exception:
error_data = {}
error_code = error_data.get("code", "")
if error_code == "context_length_exceeded":
raise OpenAIProviderError(
code=OPENAI_CONTEXT_TOO_LONG,
message="Texte trop long pour le modèle (max ~4000 tokens).",
details={"status_code": 400, "error_code": error_code},
)
if response.status_code >= 500:
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message="Service OpenAI temporairement indisponible.",
details={"status_code": response.status_code},
)
if response.status_code != 200:
error_text = response.text[:200] if response.text else "Unknown error"
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message=f"Erreur OpenAI: {error_text}",
details={"status_code": response.status_code},
)
data = response.json()
choices = data.get("choices", [])
if not choices:
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message="Erreur OpenAI: réponse vide",
details={"response": str(data)[:200]},
)
content = choices[0].get("message", {}).get("content", "")
if not content:
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message="Erreur OpenAI: réponse vide",
details={"response": str(data)[:200]},
)
usage = data.get("usage", {})
return content.strip(), usage
except Timeout:
raise OpenAIProviderError(
code=OPENAI_TIMEOUT,
message="Délai d'attente OpenAI dépassé. Le service est lent.",
details={"timeout_seconds": self._timeout},
)
except RequestsConnectionError:
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message="Service OpenAI temporairement indisponible.",
details={"error": "Connection failed"},
)
except OpenAIProviderError:
raise
except Exception as e:
error_str = str(e).lower()
if "connection" in error_str or "refused" in error_str:
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message="Service OpenAI temporairement indisponible.",
details={"original_error": str(e)[:100]},
)
raise OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message=f"Erreur OpenAI: {str(e)[:100]}",
details={"original_error": str(e)[:100]},
)
def get_name(self) -> str:
"""Return provider name."""
return self._provider_name
def is_available(self) -> bool:
"""
Check if OpenAI API is available.
Uses cached result if available and not expired.
"""
current_time = time.time()
with self._health_cache_lock:
if "is_available" in self._health_cache:
cached = self._health_cache["is_available"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
try:
url = f"{self._base_url}/models"
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(
url, headers=headers, timeout=self._health_check_timeout
)
available = response.status_code == 200
except Exception as e:
_log_warning("openai_availability_check_failed", error=str(e)[:100])
available = False
with self._health_cache_lock:
self._health_cache["is_available"] = {
"value": available,
"timestamp": current_time,
}
return available
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
"""
Translate a single text string using OpenAI LLM.
Supports custom system prompt via request.metadata["custom_prompt"].
Args:
request: TranslationRequest with text and language info
Returns:
TranslationResponse with translated text
"""
text = request.text
target_language = request.target_language
source_language = request.source_language or "auto"
if not text or not text.strip():
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
)
source_lang_name = _get_language_name(source_language)
target_lang_name = _get_language_name(target_language)
custom_prompt = None
if request.metadata:
custom_prompt = request.metadata.get("custom_prompt")
system_prompt = _build_system_prompt(
source_lang_name, target_lang_name, custom_prompt
)
last_error: Optional[OpenAIProviderError] = None
retries = 0
while retries <= self._max_retries:
try:
start_time = time.time()
result, usage = self._make_api_request(text, system_prompt)
latency = time.time() - start_time
log_kw: Dict[str, Any] = {
"chars": len(text),
"source_lang": source_language,
"target_lang": target_language,
"model": self._model,
"latency_ms": round(latency * 1000, 2),
"retries": retries,
}
if usage and isinstance(usage.get("total_tokens"), (int, float)):
log_kw["tokens_used"] = usage.get("total_tokens")
_log_info("openai_translation_success", **log_kw)
return TranslationResponse(
translated_text=result,
provider_name=self._provider_name,
from_cache=False,
source_language=source_language,
)
except OpenAIProviderError as e:
last_error = e
if e.code not in _RETRYABLE_ERRORS:
break
retries += 1
if retries <= self._max_retries:
delay = self._retry_delay * (2 ** (retries - 1))
_log_info(
"openai_translation_retry",
attempt=retries,
delay_s=round(delay, 2),
error_code=e.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
)
time.sleep(delay)
except Exception as e:
last_error = OpenAIProviderError(
code=OPENAI_SERVICE_ERROR,
message=f"Erreur OpenAI: {str(e)[:100]}",
details={"original_error": str(e)[:100]},
)
retries += 1
if retries <= self._max_retries:
delay = self._retry_delay * (2 ** (retries - 1))
time.sleep(delay)
if last_error:
_log_error(
"openai_translation_failed",
error_code=last_error.code,
text_length=len(text),
source_lang=source_language,
target_lang=target_language,
retries=retries,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error=last_error.message,
error_code=last_error.code,
error_details=last_error.details,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error="Unknown error",
error_code=OPENAI_SERVICE_ERROR,
)
def translate_batch(
self, requests: List[TranslationRequest]
) -> List[TranslationResponse]:
"""
Translate multiple texts.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
"""
if not requests:
return []
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
"""
Return health status details for the provider.
Includes cached result for efficiency.
Returns:
ProviderHealthStatus with availability, latency, and model information
"""
current_time = time.time()
with self._health_cache_lock:
if "health_check" in self._health_cache:
cached = self._health_cache["health_check"]
if current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
start_time = time.time()
last_check_iso = datetime.now(timezone.utc).isoformat()
try:
url = f"{self._base_url}/models"
headers = {"Authorization": f"Bearer {self._api_key}"}
response = requests.get(
url, headers=headers, timeout=self._health_check_timeout
)
latency_ms = (time.time() - start_time) * 1000
available = response.status_code == 200
error_msg = None
model_available = None
if available:
try:
models_data = response.json().get("data", [])
model_ids = [m.get("id", "") for m in models_data]
model_available = self._model in model_ids or any(
self._model in mid for mid in model_ids
)
except Exception:
model_available = None
else:
if response.status_code == 401:
error_msg = "Invalid API key"
else:
error_msg = f"OpenAI API returned {response.status_code}"
status = ProviderHealthStatus(
name=self._provider_name,
available=available,
latency_ms=round(latency_ms, 2),
error=error_msg,
last_check=last_check_iso,
model=self._model,
model_available=model_available,
)
except Exception as e:
latency_ms = (time.time() - start_time) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=False,
latency_ms=round(latency_ms, 2),
error=str(e)[:100],
last_check=last_check_iso,
model=self._model,
model_available=False,
)
with self._health_cache_lock:
self._health_cache["health_check"] = {
"value": status,
"timestamp": current_time,
}
return status
def register_openai_provider():
"""
Register the OpenAI provider in the global registry.
This function should be called during module initialization
to make the provider available through the registry.
"""
from .registry import registry
provider = get_openai_provider()
registry.register("openai", provider)
return provider
_provider_instance: Optional[OpenAITranslationProvider] = None
_provider_lock = threading.Lock()
def get_openai_provider() -> OpenAITranslationProvider:
"""Get or create the OpenAI provider instance (reads config from env)."""
global _provider_instance
if _provider_instance is None:
with _provider_lock:
if _provider_instance is None:
from .config import ProvidersConfig
_provider_instance = OpenAITranslationProvider(
api_key=ProvidersConfig.OPENAI_API_KEY,
model=ProvidersConfig.OPENAI_MODEL,
timeout=ProvidersConfig.OPENAI_TIMEOUT,
max_retries=ProvidersConfig.OPENAI_MAX_RETRIES,
retry_delay=ProvidersConfig.OPENAI_RETRY_DELAY,
base_url=ProvidersConfig.OPENAI_BASE_URL,
health_check_timeout=ProvidersConfig.OPENAI_HEALTH_CHECK_TIMEOUT,
)
return _provider_instance
def reset_openai_provider() -> None:
"""Reset the OpenAI provider singleton (useful when config changes)."""
global _provider_instance
with _provider_lock:
_provider_instance = None

148
services/providers/registry.py Normal file
View File

@@ -0,0 +1,148 @@
"""
Provider Registry - Singleton pattern for managing translation providers.
Provides a central registry for all translation providers with:
- Registration and retrieval by name
- Listing available providers
- Fallback chain support
"""
from typing import Dict, List, Optional
import threading
from .base import TranslationProvider
class ProviderRegistry:
"""
Singleton registry for translation providers.
Thread-safe implementation for managing multiple translation providers
with support for fallback chains.
"""
_instance: Optional["ProviderRegistry"] = None
_lock: threading.Lock = threading.Lock()
def __new__(cls) -> "ProviderRegistry":
"""Create or return the singleton instance."""
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._providers: Dict[str, TranslationProvider] = {}
cls._instance._providers_lock = threading.RLock()
return cls._instance
def register(self, name: str, provider: TranslationProvider) -> None:
"""
Register a translation provider.
Args:
name: Unique name for the provider (e.g., "google", "deepl")
provider: TranslationProvider instance
"""
with self._providers_lock:
self._providers[name] = provider
def unregister(self, name: str) -> bool:
"""
Unregister a translation provider.
Args:
name: Name of the provider to remove
Returns:
True if provider was removed, False if not found
"""
with self._providers_lock:
if name in self._providers:
del self._providers[name]
return True
return False
def get(self, name: str) -> Optional[TranslationProvider]:
"""
Get a registered provider by name.
Args:
name: Provider name
Returns:
TranslationProvider instance or None if not found
"""
with self._providers_lock:
return self._providers.get(name)
def list_all(self) -> List[str]:
"""
List all registered provider names.
Returns:
List of provider names
"""
with self._providers_lock:
return list(self._providers.keys())
def list_available(self) -> List[str]:
"""
List names of all available (reachable) providers.
Returns:
List of provider names that are currently available
"""
with self._providers_lock:
return [
name
for name, provider in self._providers.items()
if provider.is_available()
]
def get_first_available(self, names: List[str]) -> Optional[TranslationProvider]:
"""
Get the first available provider from a list of names (fallback chain).
Iterates through the list in order and returns the first provider
that is available. This enables graceful degradation when providers
are unavailable.
Args:
names: List of provider names in priority order
Returns:
First available TranslationProvider or None if all are unavailable
"""
for name in names:
provider = self.get(name)
if provider is not None and provider.is_available():
return provider
return None
def clear(self) -> None:
"""Remove all registered providers."""
with self._providers_lock:
self._providers.clear()
def __len__(self) -> int:
"""Return the number of registered providers."""
with self._providers_lock:
return len(self._providers)
def __contains__(self, name: str) -> bool:
"""Check if a provider is registered."""
with self._providers_lock:
return name in self._providers
def get_registry() -> ProviderRegistry:
"""
Get the global provider registry instance.
Returns:
The singleton ProviderRegistry instance
"""
return ProviderRegistry()
# Global registry instance
registry = ProviderRegistry()

120
services/providers/schemas.py Normal file
View File

@@ -0,0 +1,120 @@
"""
Pydantic models for translation provider request/response schemas.
"""
import re
from typing import Optional, List
from pydantic import BaseModel, ConfigDict, Field, field_validator
LANGUAGE_CODE_PATTERN = re.compile(r"^[a-z]{2}(-[A-Z]{2})?$|^auto$")
class TranslationRequest(BaseModel):
"""Request model for translation operations."""
text: str = Field(..., description="Text to translate")
target_language: str = Field(
..., description="Target language code (e.g., 'en', 'fr', 'es')"
)
source_language: str = Field(
default="auto", description="Source language code (default: auto-detect)"
)
metadata: Optional[dict] = Field(
default=None,
description="Optional metadata for provider-specific options (e.g., custom_prompt)",
)
@field_validator("target_language", "source_language")
@classmethod
def validate_language_code(cls, v: str) -> str:
if not LANGUAGE_CODE_PATTERN.match(v):
raise ValueError(
f"Invalid language code '{v}'. Expected format: 'xx' or 'xx-XX' (e.g., 'en', 'fr', 'en-US')"
)
return v
class TranslationResponse(BaseModel):
"""Response model for translation operations."""
translated_text: str = Field(..., description="Translated text")
provider_name: str = Field(
..., description="Name of the provider that performed the translation"
)
from_cache: bool = Field(
default=False, description="Whether the result came from cache"
)
source_language: Optional[str] = Field(
default=None, description="Detected or specified source language"
)
error: Optional[str] = Field(
default=None, description="Error message if translation failed"
)
error_code: Optional[str] = Field(
default=None, description="Error code for programmatic error handling"
)
error_details: Optional[dict] = Field(
default=None, description="Additional error details"
)
@property
def success(self) -> bool:
"""Check if translation was successful."""
return self.error is None
def to_error_dict(self) -> dict:
"""Convert error to dictionary format for API responses."""
if self.error is None:
return {}
result = {
"error": self.error_code or "UNKNOWN_ERROR",
"message": self.error,
}
if self.error_details:
result["details"] = self.error_details
return result
class BatchTranslationRequest(BaseModel):
"""Request model for batch translation operations."""
texts: List[str] = Field(..., description="List of texts to translate")
target_language: str = Field(..., description="Target language code")
source_language: str = Field(
default="auto", description="Source language code (default: auto-detect)"
)
class BatchTranslationResponse(BaseModel):
"""Response model for batch translation operations."""
translated_texts: List[str] = Field(..., description="List of translated texts")
provider_name: str = Field(
..., description="Name of the provider that performed the translations"
)
from_cache_count: int = Field(default=0, description="Number of results from cache")
class ProviderHealthStatus(BaseModel):
"""Health status model for a translation provider."""
model_config = ConfigDict(protected_namespaces=())
name: str = Field(..., description="Provider name")
available: bool = Field(..., description="Whether the provider is available")
latency_ms: Optional[float] = Field(
default=None, description="Response latency in milliseconds"
)
error: Optional[str] = Field(
default=None, description="Error message if unavailable"
)
last_check: Optional[str] = Field(
default=None, description="ISO timestamp of last health check"
)
model: Optional[str] = Field(
default=None, description="Model name (e.g. for LLM providers)"
)
model_available: Optional[bool] = Field(
default=None, description="Whether the configured model is available"
)

149
services/storage_tracker.py Normal file
View File

@@ -0,0 +1,149 @@
import os
import json
import logging
from datetime import datetime, timezone
from typing import Optional, Any, Dict
from config import config
try:
import structlog
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
def _log_info(event: str, **kwargs):
"""Log info with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.info(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.info(msg)
def _log_error(event: str, **kwargs):
"""Log error with structlog or standard logging compatibility."""
if _HAS_STRUCTLOG:
logger.error(event, **kwargs)
else:
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
logger.error(msg)
# Key pattern: translation:file:{job_id}
KEY_PREFIX = "translation:file"
def _get_default_ttl() -> int:
"""Get TTL from config or default to 60 minutes."""
try:
from config import config
return config.FILE_TTL_MINUTES * 60
except Exception:
return 3600 # 60 minutes default
_async_redis = None
def _get_async_redis():
"""Return async Redis client or None. Uses REDIS_URL from env."""
global _async_redis
if _async_redis is not None:
return _async_redis if _async_redis is not False else None
# Try to get from environment first
url = os.getenv("REDIS_URL", "").strip()
if not url:
_async_redis = False
return None
try:
import redis.asyncio as redis
_async_redis = redis.Redis.from_url(url, decode_responses=True)
_log_info("redis_connected", service="storage_tracker")
return _async_redis
except Exception as e:
_log_error("redis_connection_failed", service="storage_tracker", error=str(e))
_async_redis = False
return None
class StorageTracker:
"""
Tracks file locations and metadata in Redis.
Pattern: translation:file:{job_id} -> JSON metadata
"""
def __init__(self):
self._redis = None
def _redis_client(self):
if self._redis is None:
self._redis = _get_async_redis()
return self._redis
async def track_file(
self, job_id: str, metadata: Dict[str, Any], ttl: Optional[int] = None
) -> bool:
"""
Store file metadata in Redis with TTL and log the upload.
"""
if ttl is None:
ttl = _get_default_ttl()
# Ensure timestamp is present
if "timestamp" not in metadata:
metadata["timestamp"] = datetime.now(timezone.utc).isoformat()
# Log metadata (no content)
_log_info(
"file_uploaded",
job_id=job_id,
original_filename=metadata.get("original_filename"),
file_size=metadata.get("file_size"),
file_hash=metadata.get("file_hash"),
user_id=metadata.get("user_id"),
timestamp=metadata.get("timestamp"),
)
redis_client = self._redis_client()
if not redis_client:
_log_error(
"redis_not_available", job_id=job_id, hint="File tracked in logs only"
)
return False
try:
key = f"{KEY_PREFIX}:{job_id}"
await redis_client.set(key, json.dumps(metadata), ex=ttl)
_log_info("file_tracked_in_redis", job_id=job_id, ttl_seconds=ttl)
return True
except Exception as e:
_log_error("redis_track_failed", job_id=job_id, error=str(e))
return False
async def get_file_metadata(self, job_id: str) -> Optional[Dict[str, Any]]:
"""
Retrieve file metadata from Redis.
"""
redis_client = self._redis_client()
if not redis_client:
return None
try:
key = f"{KEY_PREFIX}:{job_id}"
data = await redis_client.get(key)
return json.loads(data) if data else None
except Exception as e:
_log_error("redis_get_failed", job_id=job_id, error=str(e))
return None
# Singleton for app use
storage_tracker = StorageTracker()

861
services/translation_service.py
View File

File diff suppressed because it is too large Load Diff