feat: production deployment - full update with providers, admin, glossaries, pricing, tests

Major changes across backend, frontend, infrastructure:
- Provider system with model selection (Google, DeepL, OpenAI, Ollama, Google Cloud)
- Admin panel: user management, pricing, settings
- Glossary system with CSV import/export
- Subscription and tier quota management
- Security hardening (rate limiting, API key auth, path traversal fixes)
- Docker compose for dev, prod, and IONOS deployment
- Alembic migrations for new tables
- Frontend: dashboard, pricing page, landing page, i18n (en/fr)
- Test suite and verification scripts

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Sepehr Ramezani
2026-04-25 15:01:47 +02:00
parent 2ba4fedfc8
commit 26bd096a06
1178 changed files with 136435 additions and 3047 deletions

View File

@@ -2,8 +2,9 @@
Authentication service with JWT tokens and password hashing
This service provides user authentication with automatic backend selection:
- If DATABASE_URL is configured: Uses PostgreSQL database
- Otherwise: Falls back to JSON file storage (development mode)
- When SQLAlchemy models are available: uses the same SQLite/PostgreSQL DB as the app.
- Otherwise: falls back to JSON file storage (data/users.json).
- If the DB is enabled but a user exists only in the legacy JSON file, lookups fall back to JSON.
"""
import os
@@ -37,27 +38,25 @@ except ImportError:
PASSLIB_AVAILABLE = False
logger.warning("passlib not installed. Using SHA256 fallback for password hashing.")
# Check if database is configured
# User storage: same SQLAlchemy DB as the rest of the app (SQLite by default, or PostgreSQL).
# Previously only DATABASE_URL=postgresql* enabled the DB, so dev (SQLite) used data/users.json
# while registrations could end up in translate.db — logins then failed to find users.
DATABASE_URL = os.getenv("DATABASE_URL", "")
USE_DATABASE = bool(DATABASE_URL and DATABASE_URL.startswith("postgresql"))
USE_DATABASE = False
DATABASE_AVAILABLE = False
if USE_DATABASE:
try:
from database.repositories import UserRepository
from database.connection import get_sync_session, init_db as _init_db
from database import models as db_models
try:
from database.repositories import UserRepository
from database.connection import get_sync_session, init_db as _init_db
from database import models as db_models
DATABASE_AVAILABLE = True
logger.info("Database backend enabled for authentication")
except ImportError as e:
DATABASE_AVAILABLE = False
USE_DATABASE = False
logger.warning(f"Database modules not available: {e}. Using JSON storage.")
else:
DATABASE_AVAILABLE = False
DATABASE_AVAILABLE = True
USE_DATABASE = True
logger.info(
"Using JSON file storage for authentication (DATABASE_URL not configured)"
"Database backend enabled for authentication (SQLite or PostgreSQL, same as DATABASE_URL / SQLITE_PATH)"
)
except ImportError as e:
logger.warning(f"Database modules not available: {e}. Using JSON storage.")
from models.subscription import User, UserCreate, PlanType, SubscriptionStatus, PLANS
@@ -84,29 +83,16 @@ USERS_FILE.parent.mkdir(exist_ok=True)
# Token blocklist: jti → expiry timestamp (Unix).
# Uses Redis when available (persistent across restarts), falls back to in-memory.
_revoked_jtis: dict[str, float] = {}
_redis_blocklist_client = None
def _get_blocklist_redis():
"""Return Redis client for token blocklist, or None if unavailable."""
global _redis_blocklist_client
if _redis_blocklist_client is not None:
return _redis_blocklist_client if _redis_blocklist_client is not False else None
redis_url = os.getenv("REDIS_URL", "")
if not redis_url:
_redis_blocklist_client = False
return None
try:
import redis as redis_lib
client = redis_lib.from_url(redis_url, decode_responses=True)
client.ping()
_redis_blocklist_client = client
"""Return Redis client for token blocklist from core.redis, or None if unavailable."""
from core.redis import get_sync_redis
client = get_sync_redis()
if client:
logger.info("Token blocklist using Redis (persistent across restarts)")
return client
except Exception as e:
logger.warning(f"Redis unavailable for token blocklist, using in-memory: {e}")
_redis_blocklist_client = False
return None
return client
def revoke_token_jti(jti: str, expires_at: float) -> None:
@@ -267,6 +253,22 @@ def save_users(users: Dict[str, Dict]):
json.dump(users, f, indent=2, default=str)
def _get_user_from_json_file_by_email(email: str) -> Optional[User]:
"""Legacy users.json when DB is primary but account was never migrated."""
users = load_users()
for user_data in users.values():
if user_data.get("email", "").lower() == email.lower():
return User(**user_data)
return None
def _get_user_from_json_file_by_id(user_id: str) -> Optional[User]:
users = load_users()
if user_id in users:
return User(**users[user_id])
return None
def _db_user_to_model(db_user) -> User:
"""Convert database user model to Pydantic User model"""
return User(
@@ -300,13 +302,21 @@ def get_user_by_email(email: str) -> Optional[User]:
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
from sqlalchemy.exc import OperationalError
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_email(email)
if db_user:
return _db_user_to_model(db_user)
return None
try:
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_email(email)
if db_user:
return _db_user_to_model(db_user)
return _get_user_from_json_file_by_email(email)
except OperationalError as e:
logger.warning(
"User lookup by email failed (DB): %s; trying users.json",
e,
)
return _get_user_from_json_file_by_email(email)
else:
users = load_users()
for user_data in users.values():
@@ -320,13 +330,21 @@ def get_user_by_id(user_id: str) -> Optional[User]:
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
from sqlalchemy.exc import OperationalError
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_id(user_id)
if db_user:
return _db_user_to_model(db_user)
return None
try:
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.get_by_id(user_id)
if db_user:
return _db_user_to_model(db_user)
return _get_user_from_json_file_by_id(user_id)
except OperationalError as e:
logger.warning(
"User lookup by id failed (DB): %s; trying users.json",
e,
)
return _get_user_from_json_file_by_id(user_id)
else:
users = load_users()
if user_id in users:
@@ -386,6 +404,32 @@ def authenticate_user(email: str, password: str) -> Optional[User]:
return user
def admin_set_user_password(user_id: str, plain_password: str) -> Optional[User]:
"""
Définit un nouveau mot de passe (haché) pour un utilisateur.
Couvre la base SQLAlchemy et le fichier legacy data/users.json.
"""
if len(plain_password) < 8:
raise ValueError("Le mot de passe doit contenir au moins 8 caractères")
hp = hash_password(plain_password)
if USE_DATABASE and DATABASE_AVAILABLE:
from database.connection import get_sync_session
from database.repositories import UserRepository
with get_sync_session() as session:
repo = UserRepository(session)
db_user = repo.update(user_id, hashed_password=hp)
if db_user:
return _db_user_to_model(db_user)
users = load_users()
if user_id in users:
users[user_id]["password_hash"] = hp
users[user_id]["updated_at"] = datetime.now(timezone.utc).isoformat()
save_users(users)
return User(**users[user_id])
return None
def update_user(user_id: str, updates: Dict[str, Any]) -> Optional[User]:
"""Update a user's data"""
if USE_DATABASE and DATABASE_AVAILABLE:
@@ -397,7 +441,13 @@ def update_user(user_id: str, updates: Dict[str, Any]) -> Optional[User]:
db_user = repo.update(user_id, **updates)
if db_user:
return _db_user_to_model(db_user)
return None
users = load_users()
if user_id not in users:
return None
users[user_id].update(updates)
users[user_id]["updated_at"] = datetime.now(timezone.utc).isoformat()
save_users(users)
return User(**users[user_id])
else:
users = load_users()
if user_id not in users:
@@ -562,6 +612,40 @@ def get_user_by_api_key(api_key: str) -> Optional[User]:
return None
def get_or_create_google_user(
email: str, name: str, avatar_url: Optional[str] = None
) -> User:
"""Find or create a user via Google OAuth (email as identifier)."""
if USE_DATABASE and DATABASE_AVAILABLE:
from services.auth_service_db import get_or_create_google_user as _db_impl
db_user = _db_impl(email=email, name=name, avatar_url=avatar_url)
return _db_user_to_model(db_user)
existing = get_user_by_email(email)
if existing:
updates: Dict[str, Any] = {}
if avatar_url and not existing.avatar_url:
updates["avatar_url"] = avatar_url
if not existing.email_verified:
updates["email_verified"] = True
if updates:
updated = update_user(existing.id, updates)
return updated if updated else existing
return existing
# Password must satisfy UserCreate validators (length, upper, lower, digit)
pw = f"Aa1{secrets.token_urlsafe(24)}"
user_create = UserCreate(email=email, name=name, password=pw)
user = create_user(user_create)
update_user(
user.id,
{"email_verified": True, "avatar_url": avatar_url},
)
refreshed = get_user_by_id(user.id)
return refreshed if refreshed else user
def init_database():
"""Initialize the database (call on application startup)"""
if USE_DATABASE and DATABASE_AVAILABLE:

View File

@@ -28,7 +28,7 @@ try:
except ImportError:
PASSLIB_AVAILABLE = False
from database.connection import get_db_session
from database.connection import get_sync_session
from database.repositories import UserRepository
from database.models import User, PlanType, SubscriptionStatus
from models.subscription import PLANS
@@ -113,9 +113,38 @@ def verify_token(token: str) -> Optional[Dict[str, Any]]:
return None
def get_or_create_google_user(
email: str, name: str, avatar_url: Optional[str] = None
) -> User:
"""Find existing user by email or create one for Google OAuth."""
with get_sync_session() as db:
repo = UserRepository(db)
user = repo.get_by_email(email)
if user:
updates: Dict[str, Any] = {"last_login_at": datetime.now(timezone.utc)}
if avatar_url and not user.avatar_url:
updates["avatar_url"] = avatar_url
if not user.email_verified:
updates["email_verified"] = True
updated = repo.update(user.id, **updates)
return updated if updated else user
random_password = secrets.token_urlsafe(32)
hashed = hash_password(random_password)
user = repo.create(
email=email,
name=name,
hashed_password=hashed,
tier="free",
)
updated = repo.update(user.id, email_verified=True, avatar_url=avatar_url)
return updated if updated else user
def create_user(email: str, name: str, password: str) -> User:
"""Create a new user in the database"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
# Check if email already exists
@@ -135,7 +164,7 @@ def create_user(email: str, name: str, password: str) -> User:
def authenticate_user(email: str, password: str) -> Optional[User]:
"""Authenticate user and return user object if valid"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
user = repo.get_by_email(email)
@@ -152,28 +181,28 @@ def authenticate_user(email: str, password: str) -> Optional[User]:
def get_user_by_id(user_id: str) -> Optional[User]:
"""Get user by ID from database"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
return repo.get_by_id(user_id)
def get_user_by_email(email: str) -> Optional[User]:
"""Get user by email from database"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
return repo.get_by_email(email)
def update_user(user_id: str, updates: Dict[str, Any]) -> Optional[User]:
"""Update user fields in database"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
return repo.update(user_id, **updates)
def add_credits(user_id: str, credits: int) -> bool:
"""Add credits to user account"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
result = repo.add_credits(user_id, credits)
return result is not None
@@ -181,7 +210,7 @@ def add_credits(user_id: str, credits: int) -> bool:
def use_credits(user_id: str, credits: int) -> bool:
"""Use credits from user account"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
return repo.use_credits(user_id, credits)
@@ -190,7 +219,7 @@ def increment_usage(
user_id: str, docs: int = 0, pages: int = 0, api_calls: int = 0
) -> bool:
"""Increment user usage counters"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
result = repo.increment_usage(
user_id, docs=docs, pages=pages, api_calls=api_calls
@@ -200,7 +229,7 @@ def increment_usage(
def check_usage_limits(user_id: str) -> Dict[str, Any]:
"""Check if user is within their plan limits"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
user = repo.get_by_id(user_id)
@@ -232,7 +261,7 @@ def check_usage_limits(user_id: str) -> Dict[str, Any]:
def get_user_usage_stats(user_id: str) -> Dict[str, Any]:
"""Get detailed usage statistics for a user"""
with get_db_session() as db:
with get_sync_session() as db:
repo = UserRepository(db)
user = repo.get_by_id(user_id)

View File

@@ -15,20 +15,36 @@ except ImportError:
from models.subscription import PlanType, PLANS, CREDIT_PACKAGES, SubscriptionStatus
from services.auth_service import get_user_by_id, update_user, add_credits
from services.pricing_config import (
get_subscription_line_amount_eur,
stripe_price_ids_for_plan,
)
# Stripe configuration
STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY", "")
STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET", "")
STRIPE_PUBLISHABLE_KEY = os.getenv("STRIPE_PUBLISHABLE_KEY", "")
if STRIPE_AVAILABLE and STRIPE_SECRET_KEY:
stripe.api_key = STRIPE_SECRET_KEY
def _get_stripe_secret_key() -> str:
"""Read Stripe secret key at runtime to support hot-reload/admin updates."""
return os.getenv("STRIPE_SECRET_KEY", "").strip()
def _ensure_stripe_client_configured() -> bool:
"""Configure Stripe SDK lazily with the latest key from environment."""
if not STRIPE_AVAILABLE:
return False
secret = _get_stripe_secret_key()
if not secret:
return False
stripe.api_key = secret
return True
def is_stripe_configured() -> bool:
"""Check if Stripe is properly configured"""
return STRIPE_AVAILABLE and bool(STRIPE_SECRET_KEY)
return _ensure_stripe_client_configured()
async def create_checkout_session(
@@ -47,10 +63,37 @@ async def create_checkout_session(
return {"error": "User not found"}
plan_config = PLANS[plan]
price_id = plan_config[f"stripe_price_id_{billing_period}"]
if not price_id:
return {"error": "Plan not available for purchase"}
plan_id = plan.value
mid, yid = stripe_price_ids_for_plan(plan_id)
price_id = mid if billing_period == "monthly" else yid
# If no Stripe price id is configured, fall back to inline price_data.
# This makes test-mode checkout work with only STRIPE_SECRET_KEY configured.
line_item: Dict[str, Any]
if price_id and price_id not in ("price_xxx", ""):
line_item = {"price": price_id, "quantity": 1}
else:
amount = get_subscription_line_amount_eur(plan, billing_period)
if amount in (None, "", -1) or float(amount) <= 0:
return {
"error": "Impossible de créer le checkout: tarif invalide pour ce forfait."
}
amount_cents = int(round(float(amount) * 100))
interval = "year" if billing_period == "yearly" else "month"
line_item = {
"price_data": {
"currency": "eur",
"unit_amount": amount_cents,
"recurring": {"interval": interval},
"product_data": {
"name": f"Office Translator - {plan_config.get('name', plan_id)}"
},
},
"quantity": 1,
}
try:
# Create or get Stripe customer
@@ -70,7 +113,7 @@ async def create_checkout_session(
customer=customer_id,
mode="subscription",
payment_method_types=["card"],
line_items=[{"price": price_id, "quantity": 1}],
line_items=[line_item],
success_url=success_url or f"{os.getenv('FRONTEND_URL', 'http://localhost:3000')}/dashboard?session_id={{CHECKOUT_SESSION_ID}}",
cancel_url=cancel_url or f"{os.getenv('FRONTEND_URL', 'http://localhost:3000')}/pricing",
metadata={"user_id": user_id, "plan": plan.value},
@@ -87,6 +130,50 @@ async def create_checkout_session(
return {"error": str(e)}
async def sync_checkout_session(
user_id: str,
session_id: str,
) -> Dict[str, Any]:
"""
Sync a completed Stripe Checkout session to local user state.
Useful in local/dev environments where webhooks may not reach the backend.
"""
if not is_stripe_configured():
return {"error": "Stripe not configured"}
user = get_user_by_id(user_id)
if not user:
return {"error": "User not found"}
if not session_id or not session_id.startswith("cs_"):
return {"error": "Invalid checkout session id"}
try:
session = stripe.checkout.Session.retrieve(session_id, expand=["subscription"])
except Exception as e:
return {"error": f"Unable to retrieve checkout session: {str(e)}"}
metadata = session.get("metadata", {}) or {}
session_user_id = metadata.get("user_id")
session_customer_id = session.get("customer")
# Security check: session must belong to current authenticated user.
if session_user_id and session_user_id != user_id:
return {"error": "Checkout session does not belong to current user"}
if user.stripe_customer_id and session_customer_id and session_customer_id != user.stripe_customer_id:
return {"error": "Checkout customer mismatch"}
if session.get("status") != "complete":
return {"error": "Checkout session is not completed yet"}
await handle_checkout_completed(session)
return {
"status": "synced",
"plan": metadata.get("plan"),
"session_id": session_id,
}
async def create_credits_checkout(
user_id: str,
package_index: int,
@@ -193,12 +280,25 @@ async def handle_checkout_completed(session: Dict):
# It's a subscription
plan = metadata.get("plan")
if plan:
subscription_id = session.get("subscription")
subscription_raw = session.get("subscription")
subscription_id = None
subscription_ends_at = None
if isinstance(subscription_raw, str):
subscription_id = subscription_raw
elif subscription_raw:
# Expanded subscription object (from sync path with expand=["subscription"])
subscription_id = subscription_raw.get("id")
period_end = subscription_raw.get("current_period_end")
if period_end:
subscription_ends_at = datetime.fromtimestamp(period_end).isoformat()
update_user(user_id, {
"plan": plan,
"subscription_status": SubscriptionStatus.ACTIVE.value,
"stripe_subscription_id": subscription_id,
"docs_translated_this_month": 0, # Reset on new subscription
"subscription_ends_at": subscription_ends_at,
"docs_translated_this_month": 0,
"pages_translated_this_month": 0,
})
@@ -224,6 +324,7 @@ async def handle_subscription_updated(subscription: Dict):
update_user(user_id, {
"subscription_status": status.value,
"cancel_at_period_end": subscription.get("cancel_at_period_end", False),
"subscription_ends_at": datetime.fromtimestamp(
subscription.get("current_period_end", 0)
).isoformat() if subscription.get("current_period_end") else None
@@ -256,24 +357,37 @@ async def handle_payment_failed(invoice: Dict):
async def cancel_subscription(user_id: str) -> Dict[str, Any]:
"""Cancel a user's subscription"""
"""Cancel a user's subscription at period end."""
if not is_stripe_configured():
return {"error": "Stripe not configured"}
user = get_user_by_id(user_id)
if not user or not user.stripe_subscription_id:
return {"error": "No active subscription"}
return {"error": "No active subscription found"}
try:
# Cancel at period end
subscription = stripe.Subscription.modify(
user.stripe_subscription_id,
cancel_at_period_end=True
cancel_at_period_end=True,
)
cancel_at = None
if subscription.cancel_at:
cancel_at = datetime.fromtimestamp(subscription.cancel_at).isoformat()
subscription_ends_at = None
if subscription.current_period_end:
subscription_ends_at = datetime.fromtimestamp(subscription.current_period_end).isoformat()
update_user(user_id, {
"cancel_at_period_end": True,
"subscription_ends_at": subscription_ends_at,
})
return {
"status": "canceling",
"cancel_at": datetime.fromtimestamp(subscription.cancel_at).isoformat() if subscription.cancel_at else None
"cancel_at": cancel_at,
"subscription_ends_at": subscription_ends_at,
}
except Exception as e:
return {"error": str(e)}

200
services/pricing_config.py Normal file
View File

@@ -0,0 +1,200 @@
"""
Centralisation des tarifs d'abonnement (source unique pour admin, API publique, Stripe, checkout).
Règle métier : le prix annuel est toujours dérivé du mensuel :
prix_annuel = arrondi(mensuel × 12 × YEARLY_DISCOUNT_FACTOR, 2 décimales)
avec YEARLY_DISCOUNT_FACTOR = 0,8 (20 % sur l'équivalent 12 mois au mensuel).
Les overrides admin ne peuvent pas fixer un annuel incohérent : le serveur recalcule et persiste.
"""
from __future__ import annotations
import json
import logging
import os
import re
from pathlib import Path
from typing import Any, Optional
from models.subscription import PLANS, PlanType
logger = logging.getLogger(__name__)
# Toujours relatif à la racine du projet (config.py), pas au cwd du processus —
# sinon uvicorn lancé depuis un autre dossier écrit/lit un autre fichier et les
# changements admin (ex. Starter 7 €) semblent « ne rien faire ».
from config import config as _app_config
PRICING_FILE = _app_config.BASE_DIR / "data" / "pricing_overrides.json"
# 20 % sur la facturation annuelle vs 12 × prix mensuel
YEARLY_DISCOUNT_FACTOR = 0.8
ANNUAL_DISCOUNT_PERCENT = 20
MAX_MONTHLY_PRICE_EUR = 50_000.0
MIN_MONTHLY_PRICE_EUR = 0.01
_STRIPE_PRICE_ID_RE = re.compile(r"^price_[a-zA-Z0-9]+$")
_STRIPE_SECRET_RE = re.compile(r"^sk_(test|live)_[a-zA-Z0-9]+$")
_STRIPE_PUBLISHABLE_RE = re.compile(r"^pk_(test|live)_[a-zA-Z0-9]+$")
_STRIPE_WEBHOOK_SECRET_RE = re.compile(r"^whsec_[a-zA-Z0-9]+$")
def compute_yearly_from_monthly(monthly: float) -> float:
return round(float(monthly) * 12.0 * YEARLY_DISCOUNT_FACTOR, 2)
def validate_monthly_price_eur(value: float) -> None:
v = float(value)
if v < MIN_MONTHLY_PRICE_EUR or v > MAX_MONTHLY_PRICE_EUR:
raise ValueError(
f"Le prix mensuel doit être entre {MIN_MONTHLY_PRICE_EUR} et {MAX_MONTHLY_PRICE_EUR}"
)
def validate_stripe_price_id(value: Optional[str], field_label: str) -> str:
"""Retourne une chaîne normalisée ou lève ValueError si format douteux."""
if value is None:
return ""
s = str(value).strip()
if not s:
return ""
lower = s.lower()
if "xxx" in lower or "placeholder" in lower:
raise ValueError(f"{field_label} : identifiant invalide (placeholder interdit).")
if not _STRIPE_PRICE_ID_RE.match(s):
raise ValueError(
f"{field_label} : format attendu price_… (identifiant Stripe Price)."
)
return s
def validate_stripe_secret_key(value: str) -> None:
s = value.strip()
if not _STRIPE_SECRET_RE.match(s):
raise ValueError(
"Clé secrète Stripe invalide (attendu sk_test_… ou sk_live_…)."
)
def validate_stripe_publishable_key(value: str) -> None:
s = value.strip()
if not _STRIPE_PUBLISHABLE_RE.match(s):
raise ValueError(
"Clé publique Stripe invalide (attendu pk_test_… ou pk_live_…)."
)
def validate_stripe_webhook_secret(value: str) -> None:
s = value.strip()
if not _STRIPE_WEBHOOK_SECRET_RE.match(s):
raise ValueError("Secret webhook Stripe invalide (attendu whsec_…).")
def load_pricing_overrides() -> dict[str, Any]:
try:
if PRICING_FILE.exists():
return json.loads(PRICING_FILE.read_text(encoding="utf-8"))
except Exception as e:
logger.warning("Lecture pricing_overrides impossible: %s", e)
return {}
def save_pricing_overrides(data: dict[str, Any]) -> None:
PRICING_FILE.parent.mkdir(parents=True, exist_ok=True)
PRICING_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8")
logger.info("pricing_overrides enregistré : %s", PRICING_FILE.resolve())
def reload_dotenv_from_dotenv_file() -> None:
"""
Recharge le fichier .env dans os.environ (override=True).
Nécessaire après mise à jour admin : sans cela le process garde les valeurs
du démarrage et il faudrait redémarrer le serveur.
"""
try:
from dotenv import load_dotenv
p = _app_config.BASE_DIR / ".env"
if p.exists():
load_dotenv(p, override=True)
except Exception as e:
logger.debug("reload_dotenv_from_dotenv_file: %s", e)
def apply_runtime_config_after_admin_write() -> None:
"""À appeler après sauvegarde admin (JSON +/ou .env)."""
reload_dotenv_from_dotenv_file()
def stripe_price_id_for_plan(
plan_id: str,
period: str,
overrides: Optional[dict] = None,
) -> str:
"""period: 'monthly' | 'yearly' — aligné sur STRIPE_PRICE_STARTER_MONTHLY etc."""
ov = (overrides if overrides is not None else load_pricing_overrides()).get(
plan_id, {}
)
key = f"stripe_price_id_{period}"
raw = (ov.get(key) or "").strip()
if raw:
return raw
# Fichier .env modifié par l'admin : recharger depuis le disque (pas le os.environ figé au démarrage).
reload_dotenv_from_dotenv_file()
env_key = f"STRIPE_PRICE_{plan_id.upper()}_{period.upper()}"
env_val = (os.getenv(env_key, "") or "").strip()
return env_val
def stripe_price_ids_for_plan(plan_id: str) -> tuple[str, str]:
ov = load_pricing_overrides()
return (
stripe_price_id_for_plan(plan_id, "monthly", ov),
stripe_price_id_for_plan(plan_id, "yearly", ov),
)
def get_effective_monthly_yearly(plan_id: str) -> tuple[float, float]:
"""Mensuel depuis override ou PLANS ; annuel dérivé sauf forfaits spéciaux (gratuit / entreprise)."""
try:
pt = PlanType(plan_id)
except ValueError:
return (0.0, 0.0)
base = PLANS[pt]
ov = load_pricing_overrides().get(plan_id, {})
monthly = float(ov.get("price_monthly", base["price_monthly"]))
if monthly < 0:
return (float(base["price_monthly"]), float(base["price_yearly"]))
if monthly == 0.0:
return (0.0, 0.0)
yearly = compute_yearly_from_monthly(monthly)
return (monthly, yearly)
def get_subscription_line_amount_eur(plan: PlanType, billing_period: str) -> float:
"""Montant unitaire pour une période de facturation (checkout fallback price_data)."""
plan_id = plan.value
monthly, yearly = get_effective_monthly_yearly(plan_id)
if monthly < 0:
return -1.0
if monthly == 0.0:
return 0.0
if billing_period == "yearly":
return yearly
return monthly
def normalize_plan_override_block(plan_id: str, ov: dict[str, Any]) -> dict[str, Any]:
"""Recalcule price_yearly dans un bloc override à partir de price_monthly."""
try:
pt = PlanType(plan_id)
except ValueError:
return ov
base = PLANS[pt]
monthly = float(ov.get("price_monthly", base["price_monthly"]))
ov = dict(ov)
ov["price_monthly"] = monthly
ov["price_yearly"] = compute_yearly_from_monthly(monthly)
return ov

View File

@@ -35,7 +35,7 @@ class ProvidersConfig:
Loads settings from environment variables with sensible defaults.
"""
# Google Translate (no API key required via deep_translator)
# Google Translate (no API key required via deep_translator — accès web non officiel)
GOOGLE_ENABLED: bool = (
os.getenv("GOOGLE_TRANSLATE_ENABLED", "true").lower() == "true"
)
@@ -47,6 +47,19 @@ class ProvidersConfig:
os.getenv("GOOGLE_TRANSLATE_RETRY_DELAY", "1.0")
)
# Google Cloud Translation API v2 (clé API officielle, facturable)
# Obtenir la clé : https://console.cloud.google.com → APIs & Services → Credentials
# Activer : Cloud Translation API (Basic v2) + Facturation sur le projet
GOOGLE_CLOUD_ENABLED: bool = (
os.getenv("GOOGLE_CLOUD_ENABLED", "false").lower() == "true"
)
GOOGLE_CLOUD_API_KEY: str = os.getenv("GOOGLE_CLOUD_API_KEY", "")
GOOGLE_CLOUD_TIMEOUT: int = int(os.getenv("GOOGLE_CLOUD_TIMEOUT", "30"))
GOOGLE_CLOUD_MAX_RETRIES: int = int(os.getenv("GOOGLE_CLOUD_MAX_RETRIES", "3"))
GOOGLE_CLOUD_RETRY_DELAY: float = float(
os.getenv("GOOGLE_CLOUD_RETRY_DELAY", "1.0")
)
# DeepL
DEEPL_ENABLED: bool = os.getenv("DEEPL_ENABLED", "false").lower() == "true"
DEEPL_API_KEY: str = os.getenv("DEEPL_API_KEY", "")
@@ -143,6 +156,12 @@ class ProvidersConfig:
"google": ProviderSettings(
enabled=cls.GOOGLE_ENABLED, api_key=None, base_url=None, model=None
),
"google_cloud": ProviderSettings(
enabled=cls.GOOGLE_CLOUD_ENABLED,
api_key=cls.GOOGLE_CLOUD_API_KEY if cls.GOOGLE_CLOUD_API_KEY else None,
base_url=None,
model=None,
),
"deepl": ProviderSettings(
enabled=cls.DEEPL_ENABLED,
api_key=cls.DEEPL_API_KEY if cls.DEEPL_API_KEY else None,
@@ -187,7 +206,7 @@ class ProvidersConfig:
return False
# Providers requiring API keys
providers_requiring_key = {"deepl", "openai", "openrouter"}
providers_requiring_key = {"deepl", "openai", "openrouter", "google_cloud"}
if provider_name.lower() in providers_requiring_key:
return bool(settings.api_key)

View File

@@ -21,16 +21,10 @@ from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeou
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
from core.logging import get_logger
_HAS_STRUCTLOG = True
logger = structlog.get_logger(__name__)
except ImportError:
import logging
_HAS_STRUCTLOG = False
logger = logging.getLogger(__name__)
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
def _log_info(event: str, **kwargs):

View File

@@ -14,16 +14,10 @@ Features:
from typing import List, Optional, Dict, Any
import time
try:
import structlog
from core.logging import get_logger
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
def _log_info(event: str, **kwargs):
@@ -332,6 +326,41 @@ class LegacyFallbackAdapter:
source_language: str = "auto",
batch_size: int = 50,
) -> List[str]:
if not texts:
return []
from .config import ProvidersConfig
provider_names = ProvidersConfig.get_fallback_chain(self._mode)
requests = [
TranslationRequest(
text=t,
target_language=target_language,
source_language=source_language,
)
for t in texts
]
# Try each provider once with a full batch (avoids N× fallback chain walks).
for provider_name in provider_names:
p = registry.get(provider_name)
if p is None:
continue
if not p.is_available():
continue
try:
responses = p.translate_batch(requests)
except Exception:
continue
if len(responses) != len(requests):
continue
if all(r.error is None for r in responses):
self._last_provider_used = provider_name
return [
r.translated_text if r.translated_text is not None else texts[i]
for i, r in enumerate(responses)
]
results: List[str] = []
for t in texts:
req = TranslationRequest(

View File

@@ -0,0 +1,402 @@
"""
Google Cloud Translation Provider — API officielle v2 (Basic).
Utilise l'API REST Cloud Translation v2 avec une clé API Google Cloud.
Contrairement à google_provider.py (deep_translator, accès web non officiel),
ce provider est l'implémentation officielle et facturable.
Tarification (avril 2026) :
- 500 000 caractères/mois gratuits (par projet GCP)
- Au-delà : ~$20 / million de caractères
Prérequis :
1. Activer « Cloud Translation API » dans Google Cloud Console
2. Créer une clé API (restreinte à l'API Cloud Translation)
3. Définir GOOGLE_CLOUD_API_KEY dans .env ou dans les paramètres admin
"""
import time
import threading
from typing import Any, Dict, List, Optional
import requests
from core.logging import get_logger
from .base import TranslationProvider
from .schemas import (
BatchTranslationRequest,
BatchTranslationResponse,
ProviderHealthStatus,
TranslationRequest,
TranslationResponse,
)
logger = get_logger(__name__)
_TRANSLATE_URL = "https://translation.googleapis.com/language/translate/v2"
_MAX_CHARS_PER_REQUEST = 30_000 # Cloud Translation v2 recommends ≤ 30 000 chars
# Codes d'erreur internes
GC_QUOTA_EXCEEDED = "GC_QUOTA_EXCEEDED"
GC_INVALID_KEY = "GC_INVALID_KEY"
GC_API_NOT_ENABLED = "GC_API_NOT_ENABLED"
GC_NETWORK_ERROR = "GC_NETWORK_ERROR"
GC_UNSUPPORTED_LANGUAGE = "GC_UNSUPPORTED_LANGUAGE"
GC_TEXT_TOO_LONG = "GC_TEXT_TOO_LONG"
_RETRYABLE_CODES = {GC_QUOTA_EXCEEDED, GC_NETWORK_ERROR}
# Mapping des codes de langue pour aligner avec l'API Cloud Translation
_LANG_MAP: dict[str, str] = {
"auto": "", # Cloud v2 : source vide = détection auto
"iw": "he", # hébreu
"jv": "jw", # javanais
"nb": "no", # norvégien bokmål
}
def _normalize_lang(code: str) -> str:
if not code or code == "auto":
return "" # Cloud v2 détecte automatiquement si source vide
return _LANG_MAP.get(code.lower(), code)
class GoogleCloudProviderError(Exception):
def __init__(self, code: str, message: str, details: Optional[Dict[str, Any]] = None):
self.code = code
self.message = message
self.details = details or {}
super().__init__(message)
class GoogleCloudTranslationProvider(TranslationProvider):
"""
Fournisseur Google Cloud Translation API v2 (Basic).
Utilise des requêtes REST avec clé API.
Réservé aux forfaits payants (Pro, Business, Enterprise).
"""
def __init__(
self,
api_key: str,
timeout: int = 30,
max_retries: int = 3,
retry_delay: float = 1.0,
):
if not api_key:
raise ValueError("GoogleCloudTranslationProvider requiert une clé API.")
self._api_key = api_key
self._timeout = timeout
self._max_retries = max_retries
self._retry_delay = retry_delay
self._provider_name = "google_cloud"
self._health_cache: Dict[str, Any] = {}
self._health_cache_ttl = 60
self._health_cache_lock = threading.Lock()
self._session = requests.Session()
def get_name(self) -> str:
return self._provider_name
def is_available(self) -> bool:
current_time = time.time()
with self._health_cache_lock:
cached = self._health_cache.get("is_available")
if cached and current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
try:
resp = self._session.post(
_TRANSLATE_URL,
params={"key": self._api_key},
json={"q": "hello", "target": "fr", "format": "text"},
timeout=self._timeout,
)
available = resp.ok
except Exception:
available = False
with self._health_cache_lock:
self._health_cache["is_available"] = {"value": available, "timestamp": current_time}
return available
def _call_api(self, texts: List[str], target_lang: str, source_lang: str) -> List[str]:
"""
Appel REST à Cloud Translation API v2.
Envoie plusieurs textes en une seule requête (batch natif).
Lève GoogleCloudProviderError en cas d'erreur.
"""
payload: Dict[str, Any] = {
"q": texts,
"target": target_lang,
"format": "text",
}
if source_lang: # vide = détection auto
payload["source"] = source_lang
try:
resp = self._session.post(
_TRANSLATE_URL,
params={"key": self._api_key},
json=payload,
timeout=self._timeout,
)
except requests.Timeout:
raise GoogleCloudProviderError(
GC_NETWORK_ERROR,
"Délai dépassé pour Google Cloud Translation.",
)
except requests.ConnectionError as exc:
raise GoogleCloudProviderError(
GC_NETWORK_ERROR,
"Impossible de joindre Google Cloud Translation.",
{"original_error": str(exc)[:200]},
)
if resp.status_code == 200:
translations = resp.json().get("data", {}).get("translations", [])
return [t.get("translatedText", "") for t in translations]
# Gestion des erreurs HTTP
try:
err_body = resp.json()
err_msg = err_body.get("error", {}).get("message", resp.text[:200])
err_status = err_body.get("error", {}).get("status", "")
except Exception:
err_msg = resp.text[:200]
err_status = ""
if resp.status_code in (401, 403) or "API_KEY" in err_status:
raise GoogleCloudProviderError(
GC_INVALID_KEY,
f"Clé API Google Cloud invalide ou API non activée : {err_msg}",
{"http_status": resp.status_code},
)
if resp.status_code == 429 or "QUOTA" in err_status or "RATE_LIMIT" in err_status:
raise GoogleCloudProviderError(
GC_QUOTA_EXCEEDED,
"Quota Google Cloud Translation dépassé. Réessayez plus tard.",
{"http_status": resp.status_code},
)
raise GoogleCloudProviderError(
GC_NETWORK_ERROR,
f"Erreur Google Cloud Translation HTTP {resp.status_code}: {err_msg}",
{"http_status": resp.status_code},
)
def translate_text(self, request: TranslationRequest) -> TranslationResponse:
text = request.text
target = _normalize_lang(request.target_language)
source = _normalize_lang(request.source_language or "auto")
if not text or not text.strip():
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
)
if source and source == target:
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
source_language=source,
)
last_error: Optional[GoogleCloudProviderError] = None
retries = 0
while retries <= self._max_retries:
try:
results = self._call_api([text], target, source)
translated = results[0] if results else text
logger.info(
"google_cloud_translation_success",
chars=len(text),
target_lang=target,
retries=retries,
)
return TranslationResponse(
translated_text=translated,
provider_name=self._provider_name,
from_cache=False,
)
except GoogleCloudProviderError as exc:
last_error = exc
if exc.code not in _RETRYABLE_CODES:
break
retries += 1
if retries <= self._max_retries:
time.sleep(self._retry_delay * (2 ** (retries - 1)))
except Exception as exc:
last_error = GoogleCloudProviderError(
GC_NETWORK_ERROR,
"Erreur inattendue Google Cloud Translation.",
{"original_error": str(exc)[:200]},
)
retries += 1
if retries <= self._max_retries:
time.sleep(self._retry_delay * (2 ** (retries - 1)))
logger.error(
"google_cloud_translation_failed",
error_code=last_error.code if last_error else "UNKNOWN",
chars=len(text),
target_lang=target,
)
return TranslationResponse(
translated_text=text,
provider_name=self._provider_name,
from_cache=False,
error=last_error.message if last_error else "Erreur inconnue",
error_code=last_error.code if last_error else GC_NETWORK_ERROR,
error_details=last_error.details if last_error else {},
)
def translate_batch(self, requests: List[TranslationRequest]) -> List[TranslationResponse]:
"""
Traduit plusieurs textes en utilisant le batch natif de Cloud Translation v2.
Regroupe les requêtes partageant la même paire source/cible dans un seul
appel API (meilleur rapport qualité/coût).
"""
if not requests:
return []
tgt0 = _normalize_lang(requests[0].target_language)
src0 = _normalize_lang(requests[0].source_language or "auto")
uniform = all(
_normalize_lang(r.target_language) == tgt0
and _normalize_lang(r.source_language or "auto") == src0
for r in requests
)
if uniform:
texts = [r.text for r in requests]
last_error: Optional[GoogleCloudProviderError] = None
retries = 0
while retries <= self._max_retries:
try:
results = self._call_api(texts, tgt0, src0)
logger.info(
"google_cloud_batch_success",
count=len(texts),
target_lang=tgt0,
)
return [
TranslationResponse(
translated_text=r,
provider_name=self._provider_name,
from_cache=False,
)
for r in results
]
except GoogleCloudProviderError as exc:
last_error = exc
if exc.code not in _RETRYABLE_CODES:
break
retries += 1
if retries <= self._max_retries:
time.sleep(self._retry_delay * (2 ** (retries - 1)))
except Exception as exc:
last_error = GoogleCloudProviderError(
GC_NETWORK_ERROR, str(exc)[:200]
)
retries += 1
if retries <= self._max_retries:
time.sleep(self._retry_delay * (2 ** (retries - 1)))
# Batch failed — renvoyer l'erreur pour chaque texte
err_msg = last_error.message if last_error else "Erreur inconnue"
err_code = last_error.code if last_error else GC_NETWORK_ERROR
return [
TranslationResponse(
translated_text=r.text,
provider_name=self._provider_name,
from_cache=False,
error=err_msg,
error_code=err_code,
)
for r in requests
]
# Paires source/cible hétérogènes : appel individuel
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:
current_time = time.time()
with self._health_cache_lock:
cached = self._health_cache.get("health_check")
if cached and current_time - cached["timestamp"] < self._health_cache_ttl:
return cached["value"]
from datetime import datetime, timezone
start = time.time()
last_check_iso = datetime.now(timezone.utc).isoformat()
try:
available = self.is_available()
latency_ms = (time.time() - start) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=available,
latency_ms=round(latency_ms, 2),
error=None if available else "Clé API invalide ou quota dépassé",
last_check=last_check_iso,
)
except Exception as exc:
latency_ms = (time.time() - start) * 1000
status = ProviderHealthStatus(
name=self._provider_name,
available=False,
latency_ms=round(latency_ms, 2),
error=str(exc)[:200],
last_check=last_check_iso,
)
with self._health_cache_lock:
self._health_cache["health_check"] = {"value": status, "timestamp": current_time}
return status
class LegacyGoogleCloudAdapter:
"""
Adapteur exposant GoogleCloudTranslationProvider via l'interface legacy
(.translate / .translate_batch) utilisée par translation_service.
"""
def __init__(self, api_key: str):
self._provider = GoogleCloudTranslationProvider(api_key=api_key)
self.provider_name = "google_cloud"
def translate(self, text: str, target_language: str, source_language: str = "auto") -> str:
from .schemas import TranslationRequest
resp = self._provider.translate_text(
TranslationRequest(text=text, target_language=target_language, source_language=source_language)
)
if resp.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(resp.error_code or "UNKNOWN", resp.error, resp.error_details)
return resp.translated_text
def translate_batch(
self, texts: List[str], target_language: str, source_language: str = "auto", batch_size: int = 50
) -> List[str]:
from .schemas import TranslationRequest
reqs = [
TranslationRequest(text=t, target_language=target_language, source_language=source_language)
for t in texts
]
responses = self._provider.translate_batch(reqs)
result = []
for r in responses:
if r.error:
from utils.exceptions import TranslationProviderError
raise TranslationProviderError(r.error_code or "UNKNOWN", r.error, r.error_details)
result.append(r.translated_text)
return result

View File

@@ -20,12 +20,9 @@ from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeou
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
logger = structlog.get_logger(__name__)
except ImportError:
import logging
logger = logging.getLogger(__name__)
from core.logging import get_logger
logger = get_logger(__name__)
from .base import TranslationProvider
from .schemas import (
@@ -42,6 +39,23 @@ GOOGLE_NETWORK_ERROR = "GOOGLE_NETWORK_ERROR"
GOOGLE_UNSUPPORTED_LANGUAGE = "GOOGLE_UNSUPPORTED_LANGUAGE"
GOOGLE_TEXT_TOO_LONG = "GOOGLE_TEXT_TOO_LONG"
# Align with services.translation_service.GoogleTranslationProvider (deep_translator codes)
_GOOGLE_LANG_MAP: dict[str, str] = {
"zh": "zh-CN",
"zh-cn": "zh-CN",
"zh-tw": "zh-TW",
"iw": "he",
"he": "iw",
"jv": "jw",
"nb": "no",
}
def _normalize_lang_for_google(code: str) -> str:
if not code or code == "auto":
return "auto"
return _GOOGLE_LANG_MAP.get(code, _GOOGLE_LANG_MAP.get(code.lower(), code))
_RETRYABLE_ERRORS = {GOOGLE_NETWORK_ERROR, GOOGLE_QUOTA_EXCEEDED}
@@ -121,13 +135,13 @@ class GoogleTranslationProvider(TranslationProvider):
"""Get or create a translator instance for the current thread."""
from deep_translator import GoogleTranslator
key = f"{source_language}_{target_language}"
src = _normalize_lang_for_google(source_language)
tgt = _normalize_lang_for_google(target_language)
key = f"{src}_{tgt}"
if not hasattr(self._local, "translators"):
self._local.translators = {}
if key not in self._local.translators:
self._local.translators[key] = GoogleTranslator(
source=source_language, target=target_language
)
self._local.translators[key] = GoogleTranslator(source=src, target=tgt)
return self._local.translators[key]
def _make_api_request(
@@ -377,15 +391,42 @@ class GoogleTranslationProvider(TranslationProvider):
"""
Translate multiple texts with optimized batch processing.
Args:
requests: List of TranslationRequest objects
Returns:
List of TranslationResponse objects
When all requests share the same source/target languages, delegates to
services.translation_service.GoogleTranslationProvider.translate_batch
(batched deep_translator calls). Otherwise falls back to per-item
translate_text.
"""
if not requests:
return []
tgt0 = requests[0].target_language
src0 = requests[0].source_language or "auto"
uniform = all(
r.target_language == tgt0 and (r.source_language or "auto") == src0
for r in requests
)
if uniform:
try:
from services.translation_service import (
GoogleTranslationProvider as LegacyGoogle,
)
texts = [r.text for r in requests]
outs = LegacyGoogle().translate_batch(texts, tgt0, src0)
return [
TranslationResponse(
translated_text=out,
provider_name=self._provider_name,
from_cache=False,
)
for out in outs
]
except Exception as e:
logger.warning(
"google_batch_legacy_fallback",
error_type=type(e).__name__,
)
return [self.translate_text(req) for req in requests]
def health_check(self) -> ProviderHealthStatus:

View File

@@ -21,16 +21,10 @@ from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from urllib.parse import urljoin
try:
import structlog
from core.logging import get_logger
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
def _log_info(event: str, **kwargs):

View File

@@ -19,16 +19,10 @@ import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import structlog
from core.logging import get_logger
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
import logging
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
def _log_info(event: str, **kwargs):

View File

@@ -5,14 +5,10 @@ from datetime import datetime, timezone
from typing import Optional, Any, Dict
from config import config
try:
import structlog
from core.logging import get_logger
logger = structlog.get_logger(__name__)
_HAS_STRUCTLOG = True
except ImportError:
logger = logging.getLogger(__name__)
_HAS_STRUCTLOG = False
logger = get_logger(__name__)
_HAS_STRUCTLOG = True
def _log_info(event: str, **kwargs):
@@ -40,38 +36,16 @@ KEY_PREFIX = "translation:file"
def _get_default_ttl() -> int:
"""Get TTL from config or default to 60 minutes."""
try:
from config import config
return config.FILE_TTL_MINUTES * 60
except Exception:
return 3600 # 60 minutes default
_async_redis = None
def _get_async_redis():
"""Return async Redis client or None. Uses REDIS_URL from env."""
global _async_redis
if _async_redis is not None:
return _async_redis if _async_redis is not False else None
"""Return shared async Redis client from core.redis."""
from core.redis import get_async_redis
# Try to get from environment first
url = os.getenv("REDIS_URL", "").strip()
if not url:
_async_redis = False
return None
try:
import redis.asyncio as redis
_async_redis = redis.Redis.from_url(url, decode_responses=True)
_log_info("redis_connected", service="storage_tracker")
return _async_redis
except Exception as e:
_log_error("redis_connection_failed", service="storage_tracker", error=str(e))
_async_redis = False
return None
return get_async_redis()
class StorageTracker:

View File

@@ -19,7 +19,9 @@ import random
import logging
from collections import OrderedDict
logger = logging.getLogger(__name__)
from core.logging import get_logger
logger = get_logger(__name__)
# Map language codes to full names for LLM prompts (models understand "French" better than "fr")
_LLM_LANG_NAMES = {
@@ -195,7 +197,11 @@ class TranslationProvider(ABC):
try:
return (idx, self.translate(text, target_language, source_language))
except Exception as e:
print(f"Translation error at index {idx}: {e}")
logger.warning(
"translation_error_at_index",
index=idx,
error_type=type(e).__name__,
)
return (idx, text)
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -320,10 +326,10 @@ class GoogleTranslationProvider(TranslationProvider):
if not texts_to_translate:
return results
src_norm = self._normalize_lang(source_language)
tgt_norm = self._normalize_lang(target_language)
try:
translator = GoogleTranslator(
source=source_language, target=target_language
)
translator = GoogleTranslator(source=src_norm, target=tgt_norm)
# Process in batches
translated_texts = []
@@ -349,7 +355,10 @@ class GoogleTranslationProvider(TranslationProvider):
batch_result = batch
translated_texts.extend(batch_result)
except Exception as e:
print(f"Batch translation error, falling back to individual: {e}")
logger.warning(
"batch_translation_fallback",
error_type=type(e).__name__,
)
for text in batch:
try:
translated_texts.append(translator.translate(text))
@@ -374,17 +383,17 @@ class GoogleTranslationProvider(TranslationProvider):
return results
except Exception as e:
print(f"Batch translation failed: {e}")
logger.warning("batch_translation_failed", error_type=type(e).__name__)
# Fallback to individual translations
for idx, text in zip(indices_to_translate, texts_to_translate):
try:
results[idx] = (
GoogleTranslator(
source=source_language, target=target_language
source=src_norm, target=tgt_norm
).translate(text)
or text
)
except:
except Exception:
results[idx] = text
return results
@@ -416,7 +425,7 @@ class DeepLTranslationProvider(TranslationProvider):
translator = self._get_translator(source_language, target_language)
return translator.translate(text)
except Exception as e:
print(f"Translation error: {e}")
logger.warning("translation_error", error_type=type(e).__name__)
return text
def translate_batch(
@@ -451,7 +460,7 @@ class DeepLTranslationProvider(TranslationProvider):
return results
except Exception as e:
print(f"DeepL batch error: {e}")
logger.warning("deepl_batch_error", error_type=type(e).__name__)
return [self.translate(t, target_language, source_language) for t in texts]
@@ -484,7 +493,7 @@ class LibreTranslationProvider(TranslationProvider):
translator = self._get_translator(source_language, target_language)
return translator.translate(text)
except Exception as e:
print(f"LibreTranslate error: {e}")
logger.warning("libretranslate_error", error_type=type(e).__name__)
return text
def translate_batch(
@@ -515,7 +524,7 @@ class LibreTranslationProvider(TranslationProvider):
return results
except Exception as e:
print(f"LibreTranslate batch error: {e}")
logger.warning("libretranslate_batch_error", error_type=type(e).__name__)
return texts
@@ -582,15 +591,16 @@ ADDITIONAL CONTEXT:
translated = result.get("message", {}).get("content", "").strip()
return translated if translated else text
except requests.exceptions.ConnectionError:
print(
f"Ollama error: Cannot connect to {self.base_url}. Is Ollama running?"
logger.warning(
"ollama_connection_error",
base_url=self.base_url,
)
return text
except requests.exceptions.Timeout:
print(f"Ollama error: Request timeout after 120s")
logger.warning("ollama_timeout", timeout_s=120)
return text
except Exception as e:
print(f"Ollama translation error: {e}")
logger.warning("ollama_translation_error", error_type=type(e).__name__)
return text
def translate_batch(
@@ -645,7 +655,7 @@ ADDITIONAL CONTEXT:
return data.get("models", [])
return []
except Exception as e:
print(f"Ollama list_models error: {e}")
logger.warning("ollama_list_models_error", error_type=type(e).__name__)
return []
def translate_image(self, image_path: str, target_language: str) -> str:
@@ -677,7 +687,7 @@ ADDITIONAL CONTEXT:
result = response.json()
return result.get("message", {}).get("content", "").strip()
except Exception as e:
print(f"Ollama vision translation error: {e}")
logger.warning("ollama_vision_error", error_type=type(e).__name__)
return ""
@staticmethod
@@ -689,7 +699,7 @@ ADDITIONAL CONTEXT:
models = response.json().get("models", [])
return [model["name"] for model in models]
except Exception as e:
print(f"Error listing Ollama models: {e}")
logger.warning("ollama_list_models_error", error_type=type(e).__name__)
return []
@@ -999,7 +1009,7 @@ ADDITIONAL CONTEXT AND INSTRUCTIONS:
translated = response.choices[0].message.content.strip()
return translated if translated else text
except Exception as e:
print(f"OpenAI translation error: {e}")
logger.warning("openai_translation_error", error_type=type(e).__name__)
return text
def translate_image(self, image_path: str, target_language: str) -> str:
@@ -1047,7 +1057,7 @@ ADDITIONAL CONTEXT AND INSTRUCTIONS:
return response.choices[0].message.content.strip()
except Exception as e:
print(f"OpenAI vision translation error: {e}")
logger.warning("openai_vision_error", error_type=type(e).__name__)
return ""