sepehr/office_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: homelab deployment - NPM + IONOS DNS + monitoring + NAS backup

Browse Source 
- Restructured docker-compose for Nginx Proxy Manager (no custom nginx)
- Added domain wordly.art configuration
- Added Prometheus + Grafana monitoring stack with pre-configured dashboards
- Added PostgreSQL backup script to NAS (daily/weekly/monthly rotation)
- Added alert rules for backend, system, and Docker metrics
- Updated deployment guide for NPM + IONOS DNS homelab setup
- Added marketing plan document
- PDF translator and watermark support
- Enhanced middleware, routes, and translator modules

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
sepehr
2026-05-10 11:43:28 +02:00
parent 16ac7ca2b9
commit ce8e150a61
110 changed files with 6935 additions and 4301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
middleware/rate_limiting.py
View File

@@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
class RateLimitConfig:
"""Configuration for rate limiting"""
# Requests per window
requests_per_minute: int = 30
requests_per_hour: int = 200
requests_per_day: int = 1000
requests_per_minute: int = 120
requests_per_hour: int = 1000
requests_per_day: int = 5000
# Translation-specific limits
translations_per_minute: int = 10
@@ -35,7 +35,7 @@ class RateLimitConfig:
max_total_size_per_hour_mb: int = 500
# Burst protection
burst_limit: int = 10 # Max requests in 1 second
burst_limit: int = 30 # Max requests in 1 second
# Whitelist IPs (no rate limiting)
whitelist_ips: list = field(default_factory=lambda: ["127.0.0.1", "::1"])
@@ -270,6 +270,23 @@ class RateLimitManager:
async def check_request(self, request: Request) -> tuple[bool, str, str]:
"""Check if request is allowed, return (allowed, reason, client_id)"""
client_id = self.get_client_id(request)
# Prefer user ID for authenticated users (avoids shared limits behind proxy)
# Try to extract from already-set state (auth middleware ran first)
user_id = None
if hasattr(request, "state"):
user_id = getattr(request.state, "client_id", None)
if not user_id:
# Try to get user from auth header for better per-user limiting
auth_header = request.headers.get("Authorization", "")
if auth_header.startswith("Bearer "):
# Use a hash of the token as user identifier (no decoding needed)
import hashlib
token = auth_header[7:]
user_id = f"tok:{hashlib.sha256(token.encode()).hexdigest()[:16]}"
if user_id:
client_id = user_id
self._total_requests += 1
if self.is_whitelisted(client_id):
@@ -387,6 +404,19 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
if request.url.path.startswith("/static"):
return await call_next(request)
# Skip rate limiting for lightweight GET endpoints (read-only, cacheable)
# These are config/fetch endpoints that don't consume resources
if request.method == "GET":
skip_paths = (
"/api/v1/languages",
"/api/v1/providers",
"/api/v1/auth/me",
"/api/v1/auth/usage",
"/api/v1/translations/", # status polling (uses job_id suffix)
)
if any(request.url.path.startswith(p) for p in skip_paths):
return await call_next(request)
# Check rate limit
allowed, reason, client_id = await self.manager.check_request(request)

105
middleware/tier_quota.py
View File

@@ -1,12 +1,10 @@
"""
Tier-based daily translation quota (Story 1.6).
Uses Redis sliding-window daily counter per user; fallback in-memory when Redis unavailable.
Tier-based monthly translation quota.
Uses Redis counter per user per month; fallback in-memory when Redis unavailable.
Coexists with IP-based rate limiting in rate_limiting.py.
Source of truth: Redis (key per user per UTC date) is the authority for quota enforcement.
User.daily_translation_count in DB is kept in sync on each successful translation for
reporting/analytics; reset at midnight UTC is automatic in Redis (new key per day). DB
reset can be done by a scheduled job at midnight UTC if needed.
Free tier: 2 translations per calendar month.
Paid tiers (starter/pro/business/enterprise): unlimited.
"""
from __future__ import annotations
@@ -18,65 +16,65 @@ from typing import Optional
logger = logging.getLogger(__name__)
# Free tier: 5 translations per day (UTC). Pro (and equivalent) tiers: no daily cap.
FREE_TIER_DAILY_LIMIT = 5
KEY_PREFIX = "rate_limit:daily"
# Free tier: 2 translations per calendar month.
FREE_TIER_MONTHLY_LIMIT = 2
KEY_PREFIX = "quota:monthly"
def _utc_date_str(dt: Optional[datetime] = None) -> str:
"""Current date in UTC as YYYY-MM-DD."""
def _utc_month_str(dt: Optional[datetime] = None) -> str:
"""Current month in UTC as YYYY-MM."""
t = dt or datetime.now(timezone.utc)
return t.strftime("%Y-%m-%d")
return t.strftime("%Y-%m")
def _next_midnight_utc(dt: Optional[datetime] = None) -> datetime:
"""Next midnight UTC after the given time (or now)."""
def _next_month_utc(dt: Optional[datetime] = None) -> datetime:
"""First second of next month UTC."""
now = dt or datetime.now(timezone.utc)
tomorrow = (now.date() + timedelta(days=1))
return datetime(tomorrow.year, tomorrow.month, tomorrow.day, tzinfo=timezone.utc)
if now.month == 12:
return datetime(now.year + 1, 1, 1, tzinfo=timezone.utc)
return datetime(now.year, now.month + 1, 1, tzinfo=timezone.utc)
def _seconds_until_midnight_utc(dt: Optional[datetime] = None) -> int:
"""Seconds until next midnight UTC."""
def _seconds_until_next_month(dt: Optional[datetime] = None) -> int:
"""Seconds until start of next month UTC."""
now = dt or datetime.now(timezone.utc)
return max(0, int((_next_midnight_utc(now) - now).total_seconds()))
return max(0, int((_next_month_utc(now) - now).total_seconds()))
@dataclass
class QuotaResult:
"""Result of a quota check."""
allowed: bool
remaining: int # -1 for pro (unlimited)
remaining: int # -1 for paid tiers (unlimited)
reset_at_utc: datetime
current_usage: int = 0
limit: int = FREE_TIER_DAILY_LIMIT
limit: int = FREE_TIER_MONTHLY_LIMIT
# ---------------------------------------------------------------------------
# Redis backend (shared client from core.redis)
# Redis backend
# ---------------------------------------------------------------------------
def _get_async_redis():
"""Return async Redis client or None. Uses shared client from core.redis."""
from core.redis import get_async_redis
return get_async_redis()
# ---------------------------------------------------------------------------
# In-memory fallback (per process; not shared across workers). Documented as fallback.
# In-memory fallback
# ---------------------------------------------------------------------------
_memory_usage: dict[tuple[str, str], int] = {} # (user_id, date_utc_str) -> count
_memory_usage: dict[tuple[str, str], int] = {} # (user_id, month_str) -> count
def _memory_get(user_id: str, date_str: str) -> int:
return _memory_usage.get((user_id, date_str), 0)
def _memory_get(user_id: str, month_str: str) -> int:
return _memory_usage.get((user_id, month_str), 0)
def _memory_incr(user_id: str, date_str: str) -> int:
key = (user_id, date_str)
def _memory_incr(user_id: str, month_str: str) -> int:
key = (user_id, month_str)
_memory_usage[key] = _memory_usage.get(key, 0) + 1
return _memory_usage[key]
@@ -87,28 +85,21 @@ def _memory_incr(user_id: str, date_str: str) -> int:
class TierQuotaService:
"""
Daily translation quota per user by tier.
Redis key pattern: rate_limit:daily:{user_id}:{YYYY-MM-DD}, TTL 25h.
If Redis is unavailable, uses in-memory dict (documented fallback).
Monthly translation quota per user by tier.
Redis key pattern: quota:monthly:{user_id}:{YYYY-MM}, TTL 32 days.
"""
def __init__(self):
self._redis = None # Lazy init on first use
self._redis = None
def _redis_client(self):
if self._redis is None:
self._redis = _get_async_redis()
return self._redis
def _date_str(self, dt: Optional[datetime] = None) -> str:
return _utc_date_str(dt)
async def check_quota(self, user_id: str, tier: str) -> QuotaResult:
"""
Check if user has quota for one more translation today (UTC).
tier "free" -> limit 5/day; "pro" (or equivalent) -> unlimited.
"""
reset_at = _next_midnight_utc()
"""Check monthly quota. Free = 2/month; paid = unlimited."""
reset_at = _next_month_utc()
tier_lower = (tier or "free").lower()
if tier_lower in ("pro", "business", "enterprise", "starter"):
return QuotaResult(
@@ -118,48 +109,48 @@ class TierQuotaService:
current_usage=0,
limit=0,
)
# Free tier
date_str = self._date_str()
# Free tier — monthly counter
month_str = _utc_month_str()
redis_client = self._redis_client()
if redis_client:
try:
key = f"{KEY_PREFIX}:{user_id}:{date_str}"
key = f"{KEY_PREFIX}:{user_id}:{month_str}"
count = await redis_client.get(key)
count = int(count or 0)
except Exception as e:
logger.warning("Tier quota Redis get failed: %s, using in-memory", e)
count = _memory_get(user_id, date_str)
count = _memory_get(user_id, month_str)
else:
count = _memory_get(user_id, date_str)
remaining = max(0, FREE_TIER_DAILY_LIMIT - count)
count = _memory_get(user_id, month_str)
remaining = max(0, FREE_TIER_MONTHLY_LIMIT - count)
return QuotaResult(
allowed=count < FREE_TIER_DAILY_LIMIT,
allowed=count < FREE_TIER_MONTHLY_LIMIT,
remaining=remaining,
reset_at_utc=reset_at,
current_usage=count,
limit=FREE_TIER_DAILY_LIMIT,
limit=FREE_TIER_MONTHLY_LIMIT,
)
async def increment_on_success(self, user_id: str) -> None:
"""Increment daily translation count for user (call after successful translation)."""
date_str = self._date_str()
"""Increment monthly translation count (call after successful translation)."""
month_str = _utc_month_str()
redis_client = self._redis_client()
if redis_client:
try:
key = f"{KEY_PREFIX}:{user_id}:{date_str}"
key = f"{KEY_PREFIX}:{user_id}:{month_str}"
pipe = redis_client.pipeline()
pipe.incr(key)
pipe.expire(key, 25 * 3600) # 25h so key expires after midnight UTC
pipe.expire(key, 32 * 24 * 3600) # 32 days
await pipe.execute()
return
except Exception as e:
logger.warning("Tier quota Redis increment failed: %s, using in-memory", e)
_memory_incr(user_id, date_str)
_memory_incr(user_id, month_str)
def seconds_until_reset(self) -> int:
"""Seconds until next midnight UTC (for Retry-After header)."""
return _seconds_until_midnight_utc()
"""Seconds until start of next month UTC."""
return _seconds_until_next_month()
# Singleton for app use
# Singleton
tier_quota_service = TierQuotaService()

19
middleware/validation.py
View File

@@ -57,10 +57,12 @@ class FileValidator:
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
"application/pdf": ".pdf",
}
# Magic bytes for Office Open XML files (ZIP format)
OFFICE_MAGIC_BYTES = b"PK\x03\x04"
PDF_MAGIC_BYTES = b"%PDF"
def __init__(
self,
@@ -70,7 +72,7 @@ class FileValidator:
):
self.max_size_bytes = max_size_mb * 1024 * 1024
self.max_size_mb = max_size_mb
self.allowed_extensions = allowed_extensions or {".xlsx", ".docx", ".pptx"}
self.allowed_extensions = allowed_extensions or {".xlsx", ".docx", ".pptx", ".pdf"}
self.scan_content = scan_content
async def validate_async(self, file: UploadFile) -> ValidationResult:
@@ -281,11 +283,20 @@ class FileValidator:
def _validate_magic_bytes(self, content: bytes, extension: str):
"""Validate file magic bytes match expected format"""
# All supported formats are Office Open XML (ZIP-based)
# PDF files start with %PDF
if extension.lower() == ".pdf":
if not content.startswith(self.PDF_MAGIC_BYTES):
raise ValidationError(
"File content does not match expected PDF format. "
"The file may be corrupted.",
code="invalid_file_content",
)
return
# Office files are ZIP-based
if not content.startswith(self.OFFICE_MAGIC_BYTES):
raise ValidationError(
"Le contenu du fichier ne correspond pas au format attendu. "
"Le fichier est peut-etre corrompu ou n'est pas un document Office valide.",
"File content does not match expected Office format. "
"The file may be corrupted or not a valid Office document.",
code="invalid_file_content",
)