Files
office_translator/config.py
sepehr 4d466699fd
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m26s
feat(quality): A3 — L1 LLM judge via API (5 chunks, 0.0003 USD/job)
L1 quality layer — uses a cheap LLM via the OpenAI-compatible API to
validate translation quality. Designed to be the SECOND line of defense
after L0 (script detection, length, pattern).

Architecture:
  - sampler.py — picks 5 representative chunks per job (longest first,
    skips L0-failed indices, skips too-short or identical pairs)
  - llm_judge.py — OpenAI-compatible client, binary verdict per chunk
    (accurate / fluent / correct_language / no_leaks), JSON output,
    hard timeout, defensive (never raises), cost estimation built in
  - pipeline.py — defensive wrapper that integrates both, never breaks
    a translation job, always logs a structured event

Integration:
  - 5 feature flags in config.py (QUALITY_L1_ENABLED, _LOG_ONLY, etc.)
  - QUALITY_L1_LOG_ONLY=true by default: log-only mode, verdict NEVER
    blocks or retries a job
  - Reuses the chunks extracted by L0 (no double work)
  - Passes the set of L0-failed indices so L1 doesn't re-judge them
  - Wrapped in try/except so a misconfigured L1 NEVER breaks a job

Default config: deepseek-chat via DeepSeek API
  - Cost: ~0.0003 USD per job (5 chunks)
  - Speed: typically 1-2s per call, hard ceiling at 8s
  - Easy to swap: just set L1_JUDGE_BASE_URL and L1_JUDGE_MODEL

LLM judge is intentionally a SEPARATE model from the translator
(self-evaluation bias mitigation — Meta/Stanford papers 2024-2025).

Tests:
  test_sampler.py — 9 tests covering the sampling strategy
  test_llm_judge.py — 22 tests covering init, parsing, mocked API,
    cost estimation, env factory
  test_l1_pipeline.py — 6 tests covering the wrapper
  Total new: 37 tests, all pass
  Grand total quality+format: 264 tests passing (0 regression)

  All 36 new tests + 111 L0 tests + 117 existing translator tests = 264

Phase 1 (observation) for 2 weeks. Then QUALITY_L1_LOG_ONLY=false
to enable auto-retry via the fallback chain.
2026-07-14 16:39:47 +02:00

178 lines
7.7 KiB
Python

"""
Configuration module for the Document Translation API
SaaS-ready with comprehensive settings for production deployment
"""
import os
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
class Config:
# ============== Translation Service ==============
TRANSLATION_SERVICE = os.getenv("TRANSLATION_SERVICE", "google")
DEEPL_API_KEY = os.getenv("DEEPL_API_KEY", "")
# ============== File Upload Configuration ==============
MAX_FILE_SIZE_MB = int(os.getenv("MAX_FILE_SIZE_MB", "50"))
MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
# Directories
BASE_DIR = Path(__file__).parent
UPLOAD_DIR = BASE_DIR / "uploads"
OUTPUT_DIR = BASE_DIR / "outputs"
TEMP_DIR = BASE_DIR / "temp"
LOGS_DIR = BASE_DIR / "logs"
# Supported file types
SUPPORTED_EXTENSIONS = {".xlsx", ".docx", ".pptx", ".pdf"}
# ============== Rate Limiting (SaaS) ==============
RATE_LIMIT_ENABLED = os.getenv("RATE_LIMIT_ENABLED", "true").lower() == "true"
RATE_LIMIT_PER_MINUTE = int(os.getenv("RATE_LIMIT_PER_MINUTE", "30"))
RATE_LIMIT_PER_HOUR = int(os.getenv("RATE_LIMIT_PER_HOUR", "200"))
TRANSLATIONS_PER_MINUTE = int(os.getenv("TRANSLATIONS_PER_MINUTE", "10"))
TRANSLATIONS_PER_HOUR = int(os.getenv("TRANSLATIONS_PER_HOUR", "50"))
MAX_CONCURRENT_TRANSLATIONS = int(os.getenv("MAX_CONCURRENT_TRANSLATIONS", "5"))
# ============== Cleanup Service ==============
CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "true").lower() == "true"
CLEANUP_INTERVAL_MINUTES = int(os.getenv("CLEANUP_INTERVAL_MINUTES", "5"))
FILE_TTL_MINUTES = int(os.getenv("FILE_TTL_MINUTES", "60"))
INPUT_FILE_TTL_MINUTES = int(os.getenv("INPUT_FILE_TTL_MINUTES", "30"))
OUTPUT_FILE_TTL_MINUTES = int(os.getenv("OUTPUT_FILE_TTL_MINUTES", "120"))
# Disk space thresholds
DISK_WARNING_THRESHOLD_GB = float(os.getenv("DISK_WARNING_THRESHOLD_GB", "5.0"))
DISK_CRITICAL_THRESHOLD_GB = float(os.getenv("DISK_CRITICAL_THRESHOLD_GB", "1.0"))
MAX_TOTAL_SIZE_GB = float(os.getenv("MAX_TOTAL_SIZE_GB", "10.0"))
# ============== Security ==============
ENABLE_HSTS = os.getenv("ENABLE_HSTS", "false").lower() == "true"
_CORS_ORIGINS_RAW = os.getenv("CORS_ORIGINS", "")
CORS_ORIGINS = [o.strip() for o in _CORS_ORIGINS_RAW.split(",") if o.strip()]
# Raw value for "*" / empty checks (single source of truth)
CORS_ORIGINS_RAW = _CORS_ORIGINS_RAW
MAX_REQUEST_SIZE_MB = int(os.getenv("MAX_REQUEST_SIZE_MB", "100"))
REQUEST_TIMEOUT_SECONDS = int(os.getenv("REQUEST_TIMEOUT_SECONDS", "300"))
# ============== Monitoring ==============
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
LOG_FORMAT = os.getenv("LOG_FORMAT", "json")
ENV = os.getenv("ENV", os.getenv("ENVIRONMENT", "development")).lower()
ENABLE_REQUEST_LOGGING = (
os.getenv("ENABLE_REQUEST_LOGGING", "true").lower() == "true"
)
MAX_MEMORY_PERCENT = float(os.getenv("MAX_MEMORY_PERCENT", "80"))
# ============== Quality Layer (L0) ==============
# Track A1 of the dev plan — observability only, no behavior change.
# Set to "true" to enable. Default: false (opt-in).
QUALITY_L0_ENABLED = os.getenv("QUALITY_L0_ENABLED", "false").lower() == "true"
# Number of text samples to extract from the output file for L0 analysis.
# Keep small to avoid overhead. 20 is enough to catch language confusion.
QUALITY_L0_SAMPLE_SIZE = int(os.getenv("QUALITY_L0_SAMPLE_SIZE", "20"))
# ============== Quality Layer (L1) ==============
# Track A3 of the dev plan — API-based LLM judge.
# Observability first; the verdict is logged but never used to retry
# or block a job (QUALITY_L1_LOG_ONLY=true). After 2 weeks of
# monitoring, set QUALITY_L1_LOG_ONLY=false to enable auto-retry.
QUALITY_L1_ENABLED = os.getenv("QUALITY_L1_ENABLED", "false").lower() == "true"
QUALITY_L1_LOG_ONLY = os.getenv("QUALITY_L1_LOG_ONLY", "true").lower() == "true"
# Number of chunks sampled per job. 5 is the sweet spot (cost vs coverage).
QUALITY_L1_SAMPLE_SIZE = int(os.getenv("QUALITY_L1_SAMPLE_SIZE", "5"))
# Skip the check if the document has fewer than this many chunks.
QUALITY_L1_MIN_CHUNKS = int(os.getenv("QUALITY_L1_MIN_CHUNKS", "10"))
# Hard ceiling on the L1 call (seconds). Anything longer is a skip.
QUALITY_L1_TIMEOUT_SEC = float(os.getenv("QUALITY_L1_TIMEOUT_SEC", "8.0"))
# ============== API Configuration ==============
API_TITLE = "Document Translation API"
API_VERSION = "1.0.0"
API_DESCRIPTION = """
Advanced Document Translation API with strict formatting preservation.
## Supported Formats
- Excel (.xlsx) - Preserves cell formatting, formulas, merged cells, images
- Word (.docx) - Preserves styles, tables, images, headers/footers
- PowerPoint (.pptx) - Preserves layouts, animations, embedded media
## SaaS Features
- Rate limiting per client IP
- Automatic file cleanup
- Health monitoring
- Request logging
## API Versioning
All API endpoints are versioned under /api/v1/ prefix for backward compatibility.
"""
@classmethod
def ensure_directories(cls):
"""Create necessary directories if they don't exist"""
cls.UPLOAD_DIR.mkdir(exist_ok=True, parents=True)
cls.OUTPUT_DIR.mkdir(exist_ok=True, parents=True)
cls.TEMP_DIR.mkdir(exist_ok=True, parents=True)
cls.LOGS_DIR.mkdir(exist_ok=True, parents=True)
@classmethod
def validate_required_env(cls) -> list[str]:
"""
Validate required environment variables (Story 6.6 - NFR10).
In production (ENV=production): returns list of missing required vars; app should exit if non-empty.
In development: returns [] so defaults/warnings can be used (e.g. REDIS_URL optional if rate limit off).
"""
env = os.getenv("ENV", os.getenv("ENVIRONMENT", "development")).lower()
if env != "production":
return []
missing: list[str] = []
if not os.getenv("JWT_SECRET_KEY", "").strip():
missing.append("JWT_SECRET_KEY")
if not os.getenv("ADMIN_USERNAME", "").strip():
missing.append("ADMIN_USERNAME")
admin_pass = os.getenv("ADMIN_PASSWORD", "").strip()
admin_hash = os.getenv("ADMIN_PASSWORD_HASH", "").strip()
if not admin_pass and not admin_hash:
missing.append("ADMIN_PASSWORD or ADMIN_PASSWORD_HASH")
if not os.getenv("ADMIN_TOKEN_SECRET", "").strip():
missing.append("ADMIN_TOKEN_SECRET")
rate_limit_on = os.getenv("RATE_LIMIT_ENABLED", "true").lower() == "true"
if rate_limit_on and not os.getenv("REDIS_URL", "").strip():
missing.append("REDIS_URL")
db_url = cls._get_database_url()
if not db_url:
missing.append("DATABASE_URL")
return missing
@classmethod
def _get_database_url(cls) -> str:
"""Return DATABASE_URL or build from POSTGRES_* (AC #1 - Story 6.6)."""
url = os.getenv("DATABASE_URL", "").strip()
if url:
return url
host = os.getenv("POSTGRES_HOST", "").strip()
port = os.getenv("POSTGRES_PORT", "5432").strip()
user = os.getenv("POSTGRES_USER", "").strip()
password = os.getenv("POSTGRES_PASSWORD", "").strip()
db = os.getenv("POSTGRES_DB", "").strip()
if host and user and db:
from urllib.parse import quote_plus
pw = quote_plus(password) if password else ""
return f"postgresql://{user}:{pw}@{host}:{port}/{db}"
return ""
config = Config()
# So that database/connection.py and alembic see DATABASE_URL when only POSTGRES_* is set (AC #1)
_effective_db_url = Config._get_database_url()
if Config.ENV == "production" and _effective_db_url and not os.environ.get("DATABASE_URL", "").strip():
os.environ["DATABASE_URL"] = _effective_db_url