All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
220 lines
6.5 KiB
Python
220 lines
6.5 KiB
Python
"""
|
|
Utility functions for file handling and validation
|
|
"""
|
|
|
|
import os
|
|
import uuid
|
|
import hashlib
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from fastapi import UploadFile, HTTPException
|
|
from config import config
|
|
|
|
|
|
class FileHandler:
|
|
"""Handles file operations for the translation API"""
|
|
|
|
@staticmethod
|
|
def calculate_sha256(file_path: Path) -> Optional[str]:
|
|
"""
|
|
Calculate the SHA256 hash of a file
|
|
|
|
Args:
|
|
file_path: Path to the file
|
|
|
|
Returns:
|
|
SHA256 hash string or None if error
|
|
"""
|
|
try:
|
|
if not file_path.exists():
|
|
return None
|
|
sha256_hash = hashlib.sha256()
|
|
with open(file_path, "rb") as f:
|
|
for byte_block in iter(lambda: f.read(4096), b""):
|
|
sha256_hash.update(byte_block)
|
|
return sha256_hash.hexdigest()
|
|
except Exception as e:
|
|
import logging
|
|
|
|
logging.getLogger(__name__).error(
|
|
f"SHA256 calculation failed for {file_path}: {e}"
|
|
)
|
|
return None
|
|
|
|
@staticmethod
|
|
def validate_file_extension(filename: str) -> str:
|
|
"""
|
|
Validate that the file extension is supported
|
|
|
|
Args:
|
|
filename: Name of the file
|
|
|
|
Returns:
|
|
File extension (lowercase, with dot)
|
|
|
|
Raises:
|
|
HTTPException: If file extension is not supported
|
|
"""
|
|
file_extension = Path(filename).suffix.lower()
|
|
|
|
if file_extension not in config.SUPPORTED_EXTENSIONS:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Unsupported file type. Supported types: {', '.join(config.SUPPORTED_EXTENSIONS)}",
|
|
)
|
|
|
|
return file_extension
|
|
|
|
@staticmethod
|
|
def validate_file_size(file: UploadFile) -> None:
|
|
"""
|
|
Validate that the file size is within limits
|
|
|
|
Args:
|
|
file: Uploaded file
|
|
|
|
Raises:
|
|
HTTPException: If file is too large
|
|
"""
|
|
# Get file size
|
|
file.file.seek(0, 2) # Move to end of file
|
|
file_size = file.file.tell() # Get position (file size)
|
|
file.file.seek(0) # Reset to beginning
|
|
|
|
if file_size > config.MAX_FILE_SIZE_BYTES:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"File too large. Maximum size: {config.MAX_FILE_SIZE_MB}MB",
|
|
)
|
|
|
|
@staticmethod
|
|
async def save_upload_file(file: UploadFile, destination: Path, chunk_size: int = 65536) -> Path:
|
|
"""
|
|
Save an uploaded file to disk using chunked streaming to avoid loading
|
|
the entire file into memory at once.
|
|
|
|
Args:
|
|
file: Uploaded file
|
|
destination: Path to save the file
|
|
chunk_size: Read/write chunk size in bytes (default 64KB)
|
|
|
|
Returns:
|
|
Path to the saved file
|
|
"""
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with open(destination, "wb") as buffer:
|
|
while True:
|
|
chunk = await file.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
buffer.write(chunk)
|
|
|
|
return destination
|
|
|
|
@staticmethod
|
|
def generate_unique_filename(original_filename: str, prefix: str = "") -> str:
|
|
"""
|
|
Generate a unique filename to avoid collisions
|
|
|
|
Args:
|
|
original_filename: Original filename
|
|
prefix: Optional prefix for the filename
|
|
|
|
Returns:
|
|
Unique filename
|
|
"""
|
|
file_path = Path(original_filename)
|
|
unique_id = str(uuid.uuid4())[:8]
|
|
|
|
if prefix:
|
|
return f"{prefix}_{unique_id}_{file_path.stem}{file_path.suffix}"
|
|
else:
|
|
return f"{unique_id}_{file_path.stem}{file_path.suffix}"
|
|
|
|
@staticmethod
|
|
def cleanup_file(file_path: Path) -> None:
|
|
"""
|
|
Delete a file if it exists
|
|
|
|
Args:
|
|
file_path: Path to the file to delete
|
|
"""
|
|
import logging
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
try:
|
|
if file_path.exists():
|
|
file_path.unlink()
|
|
_logger.debug(f"Deleted file: {file_path}")
|
|
except Exception as e:
|
|
_logger.warning(f"Error deleting file {file_path}: {e}")
|
|
|
|
@staticmethod
|
|
def get_file_info(file_path: Path) -> dict:
|
|
"""
|
|
Get information about a file
|
|
|
|
Args:
|
|
file_path: Path to the file
|
|
|
|
Returns:
|
|
Dictionary with file information
|
|
"""
|
|
if not file_path.exists():
|
|
return {}
|
|
|
|
stat = file_path.stat()
|
|
|
|
return {
|
|
"filename": file_path.name,
|
|
"size_bytes": stat.st_size,
|
|
"size_mb": round(stat.st_size / (1024 * 1024), 2),
|
|
"sha256": FileHandler.calculate_sha256(file_path),
|
|
"extension": file_path.suffix,
|
|
"created": stat.st_ctime,
|
|
"modified": stat.st_mtime,
|
|
}
|
|
|
|
|
|
# Global file handler instance
|
|
file_handler = FileHandler()
|
|
|
|
|
|
def validate_zip_safety(
|
|
file_path: Path,
|
|
max_compression_ratio: float = 100.0,
|
|
max_total_uncompressed_mb: int = 1024,
|
|
) -> None:
|
|
"""Reject ZIP-based documents that expand dangerously (zip bombs).
|
|
|
|
Office files (.xlsx/.docx/.pptx) are ZIP archives: a small uploaded file
|
|
can decompress to gigabytes in memory. Raises ValueError when the file
|
|
is not a readable archive, expands beyond the allowed ratio, or when the
|
|
total uncompressed size exceeds the cap.
|
|
"""
|
|
import zipfile
|
|
|
|
max_total_bytes = max_total_uncompressed_mb * 1024 * 1024
|
|
try:
|
|
with zipfile.ZipFile(file_path) as zf:
|
|
total_uncompressed = 0
|
|
for info in zf.infolist():
|
|
if info.is_dir():
|
|
continue
|
|
total_uncompressed += info.file_size
|
|
if (
|
|
info.compress_size > 0
|
|
and info.file_size / info.compress_size > max_compression_ratio
|
|
):
|
|
raise ValueError(
|
|
f"Entry '{info.filename}' expands more than "
|
|
f"{int(max_compression_ratio)}x its compressed size."
|
|
)
|
|
if total_uncompressed > max_total_bytes:
|
|
raise ValueError(
|
|
f"Archive expands beyond {max_total_uncompressed_mb} MB."
|
|
)
|
|
except zipfile.BadZipFile as e:
|
|
raise ValueError(f"Not a valid ZIP-based document: {e}")
|