sepehr
526c87348f
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
220 lines
10 KiB
Python
220 lines
10 KiB
Python
"""
|
||
Configuration module for the Document Translation API
|
||
SaaS-ready with comprehensive settings for production deployment
|
||
"""
|
||
|
||
import os
|
||
from pathlib import Path
|
||
from dotenv import load_dotenv
|
||
|
||
load_dotenv()
|
||
|
||
|
||
class Config:
|
||
# ============== Translation Service ==============
|
||
TRANSLATION_SERVICE = os.getenv("TRANSLATION_SERVICE", "google")
|
||
DEEPL_API_KEY = os.getenv("DEEPL_API_KEY", "")
|
||
|
||
|
||
# ============== File Upload Configuration ==============
|
||
MAX_FILE_SIZE_MB = int(os.getenv("MAX_FILE_SIZE_MB", "50"))
|
||
MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
|
||
|
||
# Directories
|
||
BASE_DIR = Path(__file__).parent
|
||
UPLOAD_DIR = BASE_DIR / "uploads"
|
||
OUTPUT_DIR = BASE_DIR / "outputs"
|
||
TEMP_DIR = BASE_DIR / "temp"
|
||
LOGS_DIR = BASE_DIR / "logs"
|
||
|
||
# Supported file types
|
||
SUPPORTED_EXTENSIONS = {".xlsx", ".docx", ".pptx", ".pdf"}
|
||
|
||
# ============== Rate Limiting (SaaS) ==============
|
||
RATE_LIMIT_ENABLED = os.getenv("RATE_LIMIT_ENABLED", "true").lower() == "true"
|
||
RATE_LIMIT_PER_MINUTE = int(os.getenv("RATE_LIMIT_PER_MINUTE", "30"))
|
||
RATE_LIMIT_PER_HOUR = int(os.getenv("RATE_LIMIT_PER_HOUR", "200"))
|
||
TRANSLATIONS_PER_MINUTE = int(os.getenv("TRANSLATIONS_PER_MINUTE", "10"))
|
||
TRANSLATIONS_PER_HOUR = int(os.getenv("TRANSLATIONS_PER_HOUR", "50"))
|
||
MAX_CONCURRENT_TRANSLATIONS = int(os.getenv("MAX_CONCURRENT_TRANSLATIONS", "5"))
|
||
|
||
# ============== Cleanup Service ==============
|
||
CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "true").lower() == "true"
|
||
CLEANUP_INTERVAL_MINUTES = int(os.getenv("CLEANUP_INTERVAL_MINUTES", "5"))
|
||
FILE_TTL_MINUTES = int(os.getenv("FILE_TTL_MINUTES", "60"))
|
||
INPUT_FILE_TTL_MINUTES = int(os.getenv("INPUT_FILE_TTL_MINUTES", "30"))
|
||
OUTPUT_FILE_TTL_MINUTES = int(os.getenv("OUTPUT_FILE_TTL_MINUTES", "120"))
|
||
|
||
# Disk space thresholds
|
||
DISK_WARNING_THRESHOLD_GB = float(os.getenv("DISK_WARNING_THRESHOLD_GB", "5.0"))
|
||
DISK_CRITICAL_THRESHOLD_GB = float(os.getenv("DISK_CRITICAL_THRESHOLD_GB", "1.0"))
|
||
MAX_TOTAL_SIZE_GB = float(os.getenv("MAX_TOTAL_SIZE_GB", "10.0"))
|
||
|
||
# ============== Security ==============
|
||
ENABLE_HSTS = os.getenv("ENABLE_HSTS", "false").lower() == "true"
|
||
_CORS_ORIGINS_RAW = os.getenv("CORS_ORIGINS", "")
|
||
CORS_ORIGINS = [o.strip() for o in _CORS_ORIGINS_RAW.split(",") if o.strip()]
|
||
# Raw value for "*" / empty checks (single source of truth)
|
||
CORS_ORIGINS_RAW = _CORS_ORIGINS_RAW
|
||
MAX_REQUEST_SIZE_MB = int(os.getenv("MAX_REQUEST_SIZE_MB", "100"))
|
||
REQUEST_TIMEOUT_SECONDS = int(os.getenv("REQUEST_TIMEOUT_SECONDS", "300"))
|
||
|
||
# ============== Monitoring ==============
|
||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||
LOG_FORMAT = os.getenv("LOG_FORMAT", "json")
|
||
ENV = os.getenv("ENV", os.getenv("ENVIRONMENT", "development")).lower()
|
||
ENABLE_REQUEST_LOGGING = (
|
||
os.getenv("ENABLE_REQUEST_LOGGING", "true").lower() == "true"
|
||
)
|
||
MAX_MEMORY_PERCENT = float(os.getenv("MAX_MEMORY_PERCENT", "80"))
|
||
|
||
# ============== Quality Layer (L0) ==============
|
||
# Track A1 of the dev plan — observability only, no behavior change.
|
||
# Enabled by default since 2026-08-29: log-only, never blocks a job.
|
||
QUALITY_L0_ENABLED = os.getenv("QUALITY_L0_ENABLED", "true").lower() == "true"
|
||
# Number of text samples to extract from the output file for L0 analysis.
|
||
# Keep small to avoid overhead. 20 is enough to catch language confusion.
|
||
QUALITY_L0_SAMPLE_SIZE = int(os.getenv("QUALITY_L0_SAMPLE_SIZE", "20"))
|
||
|
||
|
||
# ============== Quality Layer (L1) ==============
|
||
# Track A3 of the dev plan — API-based LLM judge.
|
||
# Observability first; the verdict is logged but never used to retry
|
||
# or block a job (QUALITY_L1_LOG_ONLY=true). After 2 weeks of
|
||
# monitoring, set QUALITY_L1_LOG_ONLY=false to enable auto-retry.
|
||
QUALITY_L1_ENABLED = os.getenv("QUALITY_L1_ENABLED", "false").lower() == "true"
|
||
QUALITY_L1_LOG_ONLY = os.getenv("QUALITY_L1_LOG_ONLY", "true").lower() == "true"
|
||
# Number of chunks sampled per job. 5 is the sweet spot (cost vs coverage).
|
||
QUALITY_L1_SAMPLE_SIZE = int(os.getenv("QUALITY_L1_SAMPLE_SIZE", "5"))
|
||
# Skip the check if the document has fewer than this many chunks.
|
||
QUALITY_L1_MIN_CHUNKS = int(os.getenv("QUALITY_L1_MIN_CHUNKS", "10"))
|
||
# Hard ceiling on the L1 call (seconds). Anything longer is a skip.
|
||
QUALITY_L1_TIMEOUT_SEC = float(os.getenv("QUALITY_L1_TIMEOUT_SEC", "8.0"))
|
||
|
||
# ============== Quality Layer (L2 — Pro tier) ==============
|
||
# Track A4 of the dev plan — STRONGER LLM judge (8 dimensions).
|
||
# Gated to Pro+ plans in the route. Default off everywhere.
|
||
# Cost: ~$0.005–$0.02/job (gpt-4o) or ~$0.001/job (gpt-4o-mini).
|
||
# Set QUALITY_L2_TIER_GATE=false to allow L2 for free tier too.
|
||
QUALITY_L2_ENABLED = os.getenv("QUALITY_L2_ENABLED", "false").lower() == "true"
|
||
QUALITY_L2_LOG_ONLY = os.getenv("QUALITY_L2_LOG_ONLY", "true").lower() == "true"
|
||
QUALITY_L2_SAMPLE_SIZE = int(os.getenv("QUALITY_L2_SAMPLE_SIZE", "15"))
|
||
QUALITY_L2_MIN_CHUNKS = int(os.getenv("QUALITY_L2_MIN_CHUNKS", "20"))
|
||
QUALITY_L2_TIMEOUT_SEC = float(os.getenv("QUALITY_L2_TIMEOUT_SEC", "20.0"))
|
||
# When true, only Pro+ plans can use L2. Otherwise, all plans can.
|
||
QUALITY_L2_TIER_GATE = os.getenv("QUALITY_L2_TIER_GATE", "true").lower() == "true"
|
||
|
||
# ============== PDF Smart-Fit (Track B3.5) ==============
|
||
# When true, the PDF translator uses a smart overflow strategy:
|
||
# 1. Try original bbox at original size
|
||
# 2. Expand bbox vertically (3x original height)
|
||
# 3. Shrink font ONCE (0.93x) with expanded bbox
|
||
# 4. Shrink font AGAIN (0.87x cumulative) with expanded bbox
|
||
# 5. For headings (font >= 14pt): never below 90% of original
|
||
# 6. For body: never below 75% of original
|
||
# 7. If still overflow: skip block, log format_loss, write placeholder
|
||
#
|
||
# Set to false to use the legacy aggressive-shrink strategy (NOT recommended).
|
||
PDF_SMART_FIT_ENABLED = os.getenv("PDF_SMART_FIT_ENABLED", "true").lower() == "true"
|
||
|
||
# ============== Scanned PDF OCR (Mistral) ==============
|
||
# Image-only PDFs have no extractable text layer. When a PDF looks
|
||
# scanned, it is routed to the Mistral OCR API to recover the text
|
||
# before translation (output: clean re-typeset PDF).
|
||
# Pricing reference: ~$1 / 1000 pages — set MISTRAL_OCR_ENABLED=false
|
||
# to disable and reject scanned PDFs with an explicit error instead.
|
||
MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY", "").strip()
|
||
MISTRAL_OCR_MODEL = os.getenv("MISTRAL_OCR_MODEL", "mistral-ocr-latest")
|
||
MISTRAL_OCR_TIMEOUT = int(os.getenv("MISTRAL_OCR_TIMEOUT", "180"))
|
||
MISTRAL_OCR_ENABLED = os.getenv("MISTRAL_OCR_ENABLED", "true").lower() == "true"
|
||
# A page with fewer extractable characters than this is text-poor; it
|
||
# counts as a scan page only when raster images also cover most of it.
|
||
SCANNED_PDF_MIN_CHARS_PER_PAGE = int(
|
||
os.getenv("SCANNED_PDF_MIN_CHARS_PER_PAGE", "100")
|
||
)
|
||
|
||
|
||
# ============== API Configuration ==============
|
||
API_TITLE = "Document Translation API"
|
||
API_VERSION = "1.0.0"
|
||
API_DESCRIPTION = """
|
||
Advanced Document Translation API with strict formatting preservation.
|
||
|
||
## Supported Formats
|
||
- Excel (.xlsx) - Preserves cell formatting, formulas, merged cells, images
|
||
- Word (.docx) - Preserves styles, tables, images, headers/footers
|
||
- PowerPoint (.pptx) - Preserves layouts, animations, embedded media
|
||
|
||
## SaaS Features
|
||
- Rate limiting per client IP
|
||
- Automatic file cleanup
|
||
- Health monitoring
|
||
- Request logging
|
||
|
||
## API Versioning
|
||
All API endpoints are versioned under /api/v1/ prefix for backward compatibility.
|
||
"""
|
||
|
||
@classmethod
|
||
def ensure_directories(cls):
|
||
"""Create necessary directories if they don't exist"""
|
||
cls.UPLOAD_DIR.mkdir(exist_ok=True, parents=True)
|
||
cls.OUTPUT_DIR.mkdir(exist_ok=True, parents=True)
|
||
cls.TEMP_DIR.mkdir(exist_ok=True, parents=True)
|
||
cls.LOGS_DIR.mkdir(exist_ok=True, parents=True)
|
||
|
||
@classmethod
|
||
def validate_required_env(cls) -> list[str]:
|
||
"""
|
||
Validate required environment variables (Story 6.6 - NFR10).
|
||
In production (ENV=production): returns list of missing required vars; app should exit if non-empty.
|
||
In development: returns [] so defaults/warnings can be used (e.g. REDIS_URL optional if rate limit off).
|
||
"""
|
||
env = os.getenv("ENV", os.getenv("ENVIRONMENT", "development")).lower()
|
||
if env != "production":
|
||
return []
|
||
missing: list[str] = []
|
||
if not os.getenv("JWT_SECRET_KEY", "").strip():
|
||
missing.append("JWT_SECRET_KEY")
|
||
if not os.getenv("ADMIN_USERNAME", "").strip():
|
||
missing.append("ADMIN_USERNAME")
|
||
admin_pass = os.getenv("ADMIN_PASSWORD", "").strip()
|
||
admin_hash = os.getenv("ADMIN_PASSWORD_HASH", "").strip()
|
||
if not admin_pass and not admin_hash:
|
||
missing.append("ADMIN_PASSWORD or ADMIN_PASSWORD_HASH")
|
||
if not os.getenv("ADMIN_TOKEN_SECRET", "").strip():
|
||
missing.append("ADMIN_TOKEN_SECRET")
|
||
rate_limit_on = os.getenv("RATE_LIMIT_ENABLED", "true").lower() == "true"
|
||
if rate_limit_on and not os.getenv("REDIS_URL", "").strip():
|
||
missing.append("REDIS_URL")
|
||
db_url = cls._get_database_url()
|
||
if not db_url:
|
||
missing.append("DATABASE_URL")
|
||
return missing
|
||
|
||
@classmethod
|
||
def _get_database_url(cls) -> str:
|
||
"""Return DATABASE_URL or build from POSTGRES_* (AC #1 - Story 6.6)."""
|
||
url = os.getenv("DATABASE_URL", "").strip()
|
||
if url:
|
||
return url
|
||
host = os.getenv("POSTGRES_HOST", "").strip()
|
||
port = os.getenv("POSTGRES_PORT", "5432").strip()
|
||
user = os.getenv("POSTGRES_USER", "").strip()
|
||
password = os.getenv("POSTGRES_PASSWORD", "").strip()
|
||
db = os.getenv("POSTGRES_DB", "").strip()
|
||
if host and user and db:
|
||
from urllib.parse import quote_plus
|
||
|
||
pw = quote_plus(password) if password else ""
|
||
return f"postgresql://{user}:{pw}@{host}:{port}/{db}"
|
||
return ""
|
||
|
||
|
||
config = Config()
|
||
|
||
# So that database/connection.py and alembic see DATABASE_URL when only POSTGRES_* is set (AC #1)
|
||
_effective_db_url = Config._get_database_url()
|
||
if Config.ENV == "production" and _effective_db_url and not os.environ.get("DATABASE_URL", "").strip():
|
||
os.environ["DATABASE_URL"] = _effective_db_url
|