sepehr/office_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Details

Browse Source 
Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
sepehr
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
translators/pdf_translator.py
View File

@@ -21,6 +21,13 @@ Fallback:
Text-only mode:
Extract text, translate, generate a clean formatted PDF via reportlab.
Scanned PDFs:
Image-only PDFs have no text layer to extract or rewrite. They are
detected up front (average extractable characters per page below
config.SCANNED_PDF_MIN_CHARS_PER_PAGE) and routed through the Mistral
OCR API (services/mistral_ocr.py) before translation; the output is a
clean re-typeset PDF (layout is not preserved — the source is images).
"""
import time
@@ -114,8 +121,15 @@ class PDFTranslator:
"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
"/app/fonts/NotoSans-Regular.ttf",
# CJK-capable fonts (target languages zh/ja/ko render as tofu with
# a Latin-only font file)
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf",
"C:/Windows/Fonts/arial.ttf",
"C:/Windows/Fonts/msyh.ttc",
"C:/Windows/Fonts/simsun.ttc",
"C:/Windows/Fonts/msgothic.ttc",
"/System/Library/Fonts/Helvetica.ttc",
]
@@ -124,6 +138,11 @@ class PDFTranslator:
self._font_path: Optional[str] = None
self._translation_stats = {"attempted": 0, "changed": 0}
self._custom_prompt: Optional[str] = None
# OCR overrides (admin settings); None → fall back to config.MISTRAL_*
self._ocr_api_key: Optional[str] = None
self._ocr_model: Optional[str] = None
self._ocr_timeout: Optional[int] = None
self._ocr_enabled: Optional[bool] = None
def set_provider(self, provider) -> None:
"""Set the translation provider."""
@@ -133,6 +152,26 @@ class PDFTranslator:
"""Set custom system prompt for LLM providers."""
self._custom_prompt = prompt
def set_ocr_config(
self,
api_key: Optional[str] = None,
model: Optional[str] = None,
timeout: Optional[int] = None,
enabled: Optional[bool] = None,
) -> None:
"""Configure the Mistral OCR call (admin settings > env defaults).
Only the values explicitly provided override the config defaults,
so the route can pass admin-configured values and let the rest fall
back to ``config.MISTRAL_*``.
"""
self._ocr_api_key = api_key
self._ocr_model = model
if timeout is not None:
self._ocr_timeout = timeout
if enabled is not None:
self._ocr_enabled = enabled
def _get_font_path(self) -> Optional[str]:
"""Resolve a Unicode-capable TTF/OTF font file."""
if self._font_path is not None:
@@ -159,6 +198,14 @@ class PDFTranslator:
output_path = Path(output_path)
self._validate_file(input_path)
# Scanned PDFs must be detected before either mode: both rely on an
# extractable text layer, which image-only pages don't have.
if self._is_scanned_pdf(input_path):
return self._translate_scanned_pdf(
input_path, output_path, target_language, source_language,
progress_callback,
)
if pdf_mode == "text_only":
return self._translate_text_only(
input_path, output_path, target_language, source_language, progress_callback
@@ -294,8 +341,22 @@ class PDFTranslator:
original, target_language, source_language
)
if translated and translated.strip():
block["translated"] = translated
translated_blocks += 1
if translated.strip() == original.strip():
# Unchanged (already in the target language, or
# the provider failed and returned the source
# text). Leave the block completely untouched —
# redacting and rewriting identical text would
# only degrade its typography (embedded fonts
# lost, everything redrawn in the substitute).
block["translated"] = None
logger.info(
"block_translation_unchanged",
page=page_num + 1,
text_preview=original[:60],
)
else:
block["translated"] = translated
translated_blocks += 1
else:
logger.warning(
"block_translation_empty",
@@ -357,6 +418,7 @@ class PDFTranslator:
return True
return False
redacted_rects: list = []
for block in blocks:
if block.get("translated"):
# Track B3.5: single redaction per block, not per sub-bbox.
@@ -371,6 +433,7 @@ class PDFTranslator:
page.add_redact_annot(block_bbox, fill=None)
else:
page.add_redact_annot(block_bbox, fill=(1, 1, 1))
redacted_rects.append(block_bbox)
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
@@ -378,7 +441,11 @@ class PDFTranslator:
# We use the original link geometry (it's unaffected by the
# redaction). URIs are preserved verbatim; only the visible
# text changes.
if page_links_before:
# Only links intersecting an actually-redacted block are
# re-inserted: when a page has no redaction at all (every block
# unchanged), the original annotations survive apply_redactions
# and re-inserting would DUPLICATE them.
if page_links_before and redacted_rects:
reinserted = 0
lost = 0
page_rect = page.rect
@@ -393,6 +460,12 @@ class PDFTranslator:
if not page_rect.intersects(from_rect):
lost += 1
continue
# Skip links outside every redacted area — their
# original annotation is still alive on the page.
if not any(
from_rect.intersects(r) for r in redacted_rects
):
continue
# Build the link insertion kwargs based on kind
if link.get("uri"):
# External URI link
@@ -700,6 +773,13 @@ class PDFTranslator:
if a.get("_no_merge") or b.get("_no_merge"):
return False
# Table cells: consecutive rows of the same column satisfy every
# geometric merge condition (same x0, similar width, small positive
# gap) but are UNRELATED values — merging them joins two cells into
# one paragraph spanning both rows and breaks the table structure.
if a.get("_is_table_cell") or b.get("_is_table_cell"):
return False
# Must have similar font size (within 20%)
if abs(a["font_size"] - b["font_size"]) > max(a["font_size"], b["font_size"]) * 0.2:
return False
@@ -812,8 +892,22 @@ class PDFTranslator:
# PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'.
# If a custom font file is available, use it via fontfile (fontname ignored).
fontname = "helv"
fontfile = font_path
# Base-14 font variant honouring the block's bold/italic flags —
# previously every block (headings included) rendered in regular.
# When a custom Unicode fontfile is used we keep it: we have no
# bold/italic variant of that file, and glyph coverage matters more
# than weight.
if font_path:
fontname = "helv"
fontfile = font_path
else:
fontname = (
"hebi" if (block.get("is_bold") and block.get("is_italic"))
else "hebo" if block.get("is_bold")
else "heit" if block.get("is_italic")
else "helv"
)
fontfile = None
# Determine if this is a heading (larger font size = more visual weight)
is_heading = target_size >= HEADING_MIN_SIZE
@@ -998,6 +1092,12 @@ class PDFTranslator:
progress_callback=None,
translate_images=translate_images,
)
# Propagate the inner Word stats so the route's sanity gate
# (attempted/changed) works on the fallback path too.
for key, value in wt.get_translation_stats().items():
self._translation_stats[key] = (
self._translation_stats.get(key, 0) + value
)
if progress_callback:
progress_callback({
@@ -1125,6 +1225,31 @@ class PDFTranslator:
pages_text.append(text)
doc.close()
translated_pages = self._translate_page_texts(
pages_text, target_language, source_language, progress_callback
)
final_path = output_path.with_suffix(".pdf")
self._generate_clean_pdf(translated_pages, final_path, target_language)
processing_time_ms = round((time.time() - start_time) * 1000, 2)
logger.info(
"pdf_text_only_success",
file_name=input_path.name,
pages=total_pages,
processing_time_ms=processing_time_ms,
)
return final_path
def _translate_page_texts(
self,
pages_text: List[str],
target_language: str,
source_language: str,
progress_callback,
) -> List[str]:
"""Translate per-page texts, keeping order; empty pages pass through."""
non_empty_indices = [i for i, t in enumerate(pages_text) if t]
if progress_callback:
@@ -1135,6 +1260,7 @@ class PDFTranslator:
})
translated_pages = list(pages_text)
total_pages = len(pages_text)
for seq, page_idx in enumerate(non_empty_indices):
text = pages_text[page_idx]
@@ -1160,19 +1286,139 @@ class PDFTranslator:
"progress_override": pct,
})
return translated_pages
# ------------------------------------------------------------------ #
# SCANNED PDFs — Mistral OCR
# ------------------------------------------------------------------ #
def _is_scanned_pdf(self, input_path: Path) -> bool:
"""True when the PDF is image-only (no usable text layer).
A page counts as a scan page when it has almost no extractable
text AND is mostly covered by raster images. The document is
considered scanned when it contains scan pages and no page with a
real text layer — this keeps sparse-but-textual PDFs (a bare
title page, a single label) on the normal layout pipeline.
"""
from config import config
try:
import fitz
except ImportError:
return False
try:
with fitz.open(str(input_path)) as doc:
if len(doc) == 0:
return False
has_text_page = False
has_scan_page = False
for page in doc:
if len(page.get_text("text").strip()) >= config.SCANNED_PDF_MIN_CHARS_PER_PAGE:
has_text_page = True
continue
# Text-poor page: a scan page only when raster images
# cover most of its area (a bare title page has none).
page_area = abs(page.rect)
img_area = 0.0
for img in page.get_images(full=True):
for rect in page.get_image_rects(img[0]):
img_area += abs(rect & page.rect)
if page_area and img_area / page_area >= 0.5:
has_scan_page = True
except Exception as e:
logger.warning("scanned_pdf_detection_failed", error=str(e))
return False
scanned = has_scan_page and not has_text_page
if scanned:
logger.info(
"scanned_pdf_detected",
file=input_path.name,
)
return scanned
def _translate_scanned_pdf(
self,
input_path: Path,
output_path: Path,
target_language: str,
source_language: str,
progress_callback,
) -> Path:
"""OCR (Mistral) → translate → clean re-typeset PDF.
The source pages are images, so the original layout cannot be
rewritten in place; the output carries the recovered text in a
clean document instead.
"""
from config import config
from services.mistral_ocr import MistralOCRClient, MistralOCRError
# Resolution order: admin settings (via set_ocr_config) > env/config.
api_key = (self._ocr_api_key or "").strip() or config.MISTRAL_API_KEY
model = (self._ocr_model or "").strip() or config.MISTRAL_OCR_MODEL
timeout = self._ocr_timeout or config.MISTRAL_OCR_TIMEOUT
ocr_enabled = (
config.MISTRAL_OCR_ENABLED
if self._ocr_enabled is None
else self._ocr_enabled
)
if not ocr_enabled or not api_key:
raise RuntimeError(
"PDF scanné détecté (pages image sans couche texte). "
"La traduction des PDF scannés nécessite l'OCR Mistral : "
"configurez MISTRAL_API_KEY (ou fournissez un PDF avec du texte sélectionnable)."
)
start_time = time.time()
client = MistralOCRClient(
api_key=api_key,
model=model,
timeout=timeout,
)
pages_markdown = client.extract_pdf_text(
input_path, progress_callback=progress_callback
)
pages_text = [self._markdown_to_text(md) for md in pages_markdown]
translated_pages = self._translate_page_texts(
pages_text, target_language, source_language, progress_callback
)
final_path = output_path.with_suffix(".pdf")
self._generate_clean_pdf(translated_pages, final_path, target_language)
processing_time_ms = round((time.time() - start_time) * 1000, 2)
logger.info(
"pdf_text_only_success",
"pdf_scanned_success",
file_name=input_path.name,
pages=total_pages,
pages=len(pages_text),
processing_time_ms=processing_time_ms,
)
return final_path
@staticmethod
def _markdown_to_text(markdown: str) -> str:
"""Flatten OCR markdown to plain text (drop images/links/markup)."""
import re
if not markdown:
return ""
text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", markdown) # images
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) # links → label
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) # headings
text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) # bullets
# Markdown table rows → plain line of cells
text = re.sub(r"^\s*\|", "", text, flags=re.MULTILINE)
text = text.replace("|", " ")
text = re.sub(r"^\s*[-:| ]+\s*$", "", text, flags=re.MULTILINE) # rules
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def _generate_clean_pdf(
self, pages_text: List[str], output_path: Path, target_language: str = "en"
) -> None:
@@ -1305,18 +1551,29 @@ class PDFTranslator:
def _translate_single(
self, text: str, target_language: str, source_language: str
) -> str:
"""Translate a single text string."""
"""Translate a single text string.
Also feeds the job-level attempted/changed stats so the route can
detect a total provider failure (changed == 0) on PDFs too.
"""
if text and text.strip():
self._translation_stats["attempted"] += 1
if self._provider is not None:
try:
results = self._translate_with_provider([text], target_language, source_language)
if results and results[0].strip():
if results[0].strip() != text.strip():
self._translation_stats["changed"] += 1
return results[0]
except Exception as e:
logger.warning("provider_single_failed", error=str(e))
from services.translation_service import translation_service
try:
return translation_service.translate_text(text, target_language, source_language)
result = translation_service.translate_text(text, target_language, source_language)
if result and result.strip() and result.strip() != text.strip():
self._translation_stats["changed"] += 1
return result
except Exception as e:
logger.warning("legacy_single_failed", error=str(e))
return text