feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
This commit is contained in:
216
services/mistral_ocr.py
Normal file
216
services/mistral_ocr.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
Mistral OCR client — text extraction for scanned PDFs.
|
||||
|
||||
Image-only PDFs have no extractable text layer: PyMuPDF sees empty pages
|
||||
and the layout-preserving pipeline would output an empty document. This
|
||||
client calls the Mistral OCR API to recover the text.
|
||||
|
||||
API reference (2026): POST https://api.mistral.ai/v1/ocr with Bearer auth,
|
||||
body {"model", "document": {"type": "document_url", "document_url":
|
||||
"data:application/pdf;base64,..."}, "pages": [0, 1, ...]}. The response
|
||||
contains {"pages": [{"index", "markdown", "dimensions"}, ...]}.
|
||||
|
||||
The document is sent in chunks of PAGES_PER_REQUEST pages: it bounds the
|
||||
request payload and lets us report progress page by page.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
MISTRAL_OCR_URL = "https://api.mistral.ai/v1/ocr"
|
||||
DEFAULT_MODEL = "mistral-ocr-latest"
|
||||
PAGES_PER_REQUEST = 8
|
||||
|
||||
MISTRAL_INVALID_KEY = "MISTRAL_INVALID_KEY"
|
||||
MISTRAL_QUOTA_EXCEEDED = "MISTRAL_QUOTA_EXCEEDED"
|
||||
MISTRAL_TIMEOUT = "MISTRAL_TIMEOUT"
|
||||
MISTRAL_SERVICE_ERROR = "MISTRAL_SERVICE_ERROR"
|
||||
|
||||
|
||||
class MistralOCRError(Exception):
|
||||
"""Raised when the Mistral OCR API cannot extract the PDF text."""
|
||||
|
||||
def __init__(
|
||||
self, code: str, message: str, details: Optional[Dict[str, Any]] = None
|
||||
):
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class MistralOCRClient:
|
||||
"""Thin synchronous client around the Mistral OCR endpoint."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
model: str = DEFAULT_MODEL,
|
||||
timeout: int = 180,
|
||||
max_retries: int = 2,
|
||||
retry_delay: float = 2.0,
|
||||
):
|
||||
self._api_key = (api_key or "").strip()
|
||||
self._model = model or DEFAULT_MODEL
|
||||
self._timeout = timeout
|
||||
self._max_retries = max_retries
|
||||
self._retry_delay = retry_delay
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""True when an API key is configured."""
|
||||
return bool(self._api_key)
|
||||
|
||||
def _post_ocr(self, data_uri: str, pages: List[int]) -> List[Dict[str, Any]]:
|
||||
"""POST one OCR request for the given 0-based page list, with retries."""
|
||||
payload = {
|
||||
"model": self._model,
|
||||
"document": {"type": "document_url", "document_url": data_uri},
|
||||
"pages": pages,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
last_error: Optional[Exception] = None
|
||||
for attempt in range(self._max_retries + 1):
|
||||
try:
|
||||
response = requests.post(
|
||||
MISTRAL_OCR_URL,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise MistralOCRError(
|
||||
MISTRAL_INVALID_KEY,
|
||||
"Clé API Mistral invalide (MISTRAL_API_KEY).",
|
||||
{"status_code": 401},
|
||||
)
|
||||
if response.status_code in (402, 429):
|
||||
raise MistralOCRError(
|
||||
MISTRAL_QUOTA_EXCEEDED,
|
||||
"Quota Mistral OCR épuisé ou limite de débit atteinte.",
|
||||
{"status_code": response.status_code},
|
||||
)
|
||||
if response.status_code >= 500:
|
||||
raise MistralOCRError(
|
||||
MISTRAL_SERVICE_ERROR,
|
||||
f"Service Mistral OCR indisponible (HTTP {response.status_code}).",
|
||||
{"status_code": response.status_code},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise MistralOCRError(
|
||||
MISTRAL_SERVICE_ERROR,
|
||||
f"Erreur Mistral OCR (HTTP {response.status_code}): {response.text[:200]}",
|
||||
{"status_code": response.status_code},
|
||||
)
|
||||
|
||||
pages_out = response.json().get("pages", [])
|
||||
if not pages_out:
|
||||
raise MistralOCRError(
|
||||
MISTRAL_SERVICE_ERROR,
|
||||
"Réponse Mistral OCR vide.",
|
||||
)
|
||||
return pages_out
|
||||
|
||||
except MistralOCRError as e:
|
||||
if e.code in (MISTRAL_INVALID_KEY, MISTRAL_QUOTA_EXCEEDED):
|
||||
raise # not transient
|
||||
last_error = e
|
||||
except requests.exceptions.Timeout as e:
|
||||
last_error = MistralOCRError(
|
||||
MISTRAL_TIMEOUT,
|
||||
f"Délai d'attente Mistral OCR dépassé ({self._timeout}s).",
|
||||
)
|
||||
except requests.exceptions.RequestException as e:
|
||||
last_error = MistralOCRError(
|
||||
MISTRAL_SERVICE_ERROR,
|
||||
f"Mistral OCR injoignable: {str(e)[:150]}",
|
||||
)
|
||||
|
||||
if attempt < self._max_retries:
|
||||
delay = self._retry_delay * (2**attempt)
|
||||
logger.info(
|
||||
"mistral_ocr_retry",
|
||||
attempt=attempt + 1,
|
||||
delay_s=round(delay, 2),
|
||||
error=last_error.code if last_error else "unknown",
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
raise last_error or MistralOCRError(
|
||||
MISTRAL_SERVICE_ERROR, "Erreur Mistral OCR inconnue."
|
||||
)
|
||||
|
||||
def extract_pdf_text(
|
||||
self,
|
||||
pdf_path: Path,
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
) -> List[str]:
|
||||
"""OCR a PDF file and return one text (markdown) string per page.
|
||||
|
||||
Pages are processed in chunks of ``PAGES_PER_REQUEST``; the returned
|
||||
list is ordered by page index, empty strings for pages OCR returned
|
||||
nothing for.
|
||||
"""
|
||||
import fitz
|
||||
|
||||
pdf_path = Path(pdf_path)
|
||||
data_b64 = base64.b64encode(pdf_path.read_bytes()).decode("ascii")
|
||||
data_uri = f"data:application/pdf;base64,{data_b64}"
|
||||
|
||||
with fitz.open(str(pdf_path)) as doc:
|
||||
total_pages = len(doc)
|
||||
if total_pages == 0:
|
||||
raise MistralOCRError(MISTRAL_SERVICE_ERROR, "PDF vide (0 page).")
|
||||
|
||||
chunks = [
|
||||
list(range(start, min(start + PAGES_PER_REQUEST, total_pages)))
|
||||
for start in range(0, total_pages, PAGES_PER_REQUEST)
|
||||
]
|
||||
|
||||
page_texts: List[str] = [""] * total_pages
|
||||
done_chunks = 0
|
||||
for chunk in chunks:
|
||||
pages_out = self._post_ocr(data_uri, chunk)
|
||||
for page in pages_out:
|
||||
idx = int(page.get("index", -1))
|
||||
if 0 <= idx < total_pages:
|
||||
page_texts[idx] = page.get("markdown", "") or ""
|
||||
|
||||
done_chunks += 1
|
||||
logger.info(
|
||||
"mistral_ocr_chunk_done",
|
||||
pages_done=min(done_chunks * PAGES_PER_REQUEST, total_pages),
|
||||
total_pages=total_pages,
|
||||
)
|
||||
if progress_callback and chunks:
|
||||
pct = int(5 + 20 * done_chunks / len(chunks))
|
||||
progress_callback(
|
||||
{
|
||||
"current": done_chunks,
|
||||
"total": len(chunks),
|
||||
"phase": f"OCR (Mistral) {min(done_chunks * PAGES_PER_REQUEST, total_pages)}/{total_pages}",
|
||||
"paragraph": done_chunks,
|
||||
"total_paragraphs": len(chunks),
|
||||
"progress_override": pct,
|
||||
}
|
||||
)
|
||||
|
||||
extracted = sum(1 for t in page_texts if t.strip())
|
||||
logger.info(
|
||||
"mistral_ocr_extracted",
|
||||
pages_with_text=extracted,
|
||||
total_pages=total_pages,
|
||||
)
|
||||
return page_texts
|
||||
Reference in New Issue
Block a user