feat(rtl): rendu droite-a-gauche complet pour Word, PowerPoint, Excel et PDF
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m37s

- source unique RTL_LANGUAGES/is_rtl dans core/languages.py (fin des 3 copies)
- Word: bidi partout (corps, tableaux bidiVisual, notes/fin/commentaires,
  zones de texte, 6 zones d'en-tetes/pieds), insertion OOXML ordonnee,
  polices cs elargies aux 11 langues
- PowerPoint: alignements explicites preserves, notes du presentateur,
  indice de police <a:cs> insert a sa place
- Excel: feuilles affichees de droite a gauche, feuilles graphiques ignorees
- PDF: faconnage bidi (arabic-reshaper + python-bidi), polices par ecriture
  (arabe/hebreu), TTF enregistree pour le PDF recompose
- 46 tests nouveaux (tests/test_translators/test_rtl_layout.py), 253 au total
This commit is contained in:
2026-09-01 20:22:46 +02:00
parent fddd7b7428
commit f22f645fab
12 changed files with 1727 additions and 80 deletions

View File

@@ -33,10 +33,12 @@ Scanned PDFs:
import time
import shutil
import subprocess
import zlib
from pathlib import Path
from typing import Dict, Any, Optional, Callable, List
from core.logging import get_logger
from core.languages import is_rtl
logger = get_logger(__name__)
@@ -60,8 +62,70 @@ HEADING_MIN_SCALE = 0.90
# Maximum font shrink for body text
BODY_MIN_SCALE = 0.75
# RTL language codes
RTL_LANGUAGES = frozenset({"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"})
# Warn at most once when the RTL shaping libraries are missing.
_RTL_SHAPING_MISSING_WARNED = False
def _shape_rtl(text: str) -> str:
"""Shape Arabic-script text for PDF engines that do no bidi of their own.
arabic-reshaper joins the letters into their contextual forms and
python-bidi reorders the string for visual (left-to-right) rendering.
Both libraries are optional on purpose: when they are missing or
fail, the text is returned unchanged with a warning — the
translation itself must always succeed, only the rendering degrades.
"""
global _RTL_SHAPING_MISSING_WARNED
if not text:
return text
try:
import arabic_reshaper
from bidi.algorithm import get_display
except Exception:
if not _RTL_SHAPING_MISSING_WARNED:
logger.warning(
"rtl_shaping_libs_missing",
hint="pip install arabic-reshaper python-bidi for correct RTL rendering",
)
_RTL_SHAPING_MISSING_WARNED = True
return text
try:
return get_display(arabic_reshaper.reshape(text))
except Exception as e:
logger.warning("rtl_shaping_failed", error=str(e)[:200])
return text
# Font path → reportlab font name, for RTL fonts registered in this
# process. A fixed name would make two consecutive translations with
# different fonts (e.g. Arabic then Hebrew) overwrite each other's
# registration.
_RTL_FONT_REGISTRATIONS: Dict[str, str] = {}
def _register_rtl_font(font_path: str) -> str:
"""Register a TTF under a stable per-path name, once per process.
Returns the reportlab font name, or "Helvetica" when registration
fails (warning logged — a font problem must never fail the job).
"""
registered = _RTL_FONT_REGISTRATIONS.get(font_path)
if registered:
return registered
try:
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
digest = format(zlib.crc32(str(font_path).encode("utf-8")) & 0xFFFFFFFF, "08x")
name = f"WordlyRTL-{digest}"
pdfmetrics.registerFont(TTFont(name, font_path))
_RTL_FONT_REGISTRATIONS[font_path] = name
return name
except Exception as e:
logger.warning(
"pdf_rtl_font_register_failed", error=str(e)[:200], font=font_path
)
return "Helvetica"
def _record_format_loss_metric(element_type: str, count: int = 1) -> None:
@@ -109,6 +173,29 @@ def _libreoffice_available() -> bool:
_libreoffice_available._cache = None # type: ignore[attr-defined]
# Script groups among the RTL languages — the preferred font depends on
# the writing system, not just the direction: a Hebrew target would
# render as tofu with an Arabic-only font.
_ARABIC_SCRIPT_LANGUAGES = frozenset(
{"ar", "fa", "ur", "ps", "ku", "sd", "ug", "ckb"}
)
_HEBREW_SCRIPT_LANGUAGES = frozenset({"he", "yi"})
def _rtl_script(code: str) -> str:
"""Script group of an RTL language code: "arabic", "hebrew" or ""
("" meaning: use the generic Unicode font list — e.g. Thaana "dv",
for which no dedicated font is searched).
"""
base = (code or "").strip().replace("_", "-").split("-")[0].lower()
if base in _ARABIC_SCRIPT_LANGUAGES:
return "arabic"
if base in _HEBREW_SCRIPT_LANGUAGES:
return "hebrew"
return ""
class PDFTranslator:
"""Translates PDF files with layout preservation using PyMuPDF."""
@@ -133,9 +220,37 @@ class PDFTranslator:
"/System/Library/Fonts/Helvetica.ttc",
]
# Arabic-script-capable fonts (ar, fa, ur, ps, ku, sd, ug, ckb),
# searched FIRST for those targets. Debian's fonts-noto package
# ships Noto Naskh Arabic and Noto Sans Arabic under
# /usr/share/fonts/truetype/noto/; on Windows, Arial covers the
# whole Arabic script.
_RTL_ARABIC_FONT_SEARCH_PATHS = [
"/usr/share/fonts/truetype/noto/NotoNaskhArabic-Regular.ttf",
"/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf",
"/usr/share/fonts/opentype/noto/NotoNaskhArabic-Regular.ttf",
"/usr/share/fonts/opentype/noto/NotoSansArabic-Regular.ttf",
"/app/fonts/NotoNaskhArabic-Regular.ttf",
"/app/fonts/NotoSansArabic-Regular.ttf",
"C:/Windows/Fonts/arial.ttf",
]
# Hebrew-script-capable fonts (he, yi), searched FIRST for those
# targets. Same Debian package provides Noto Sans Hebrew; Arial
# covers Hebrew on Windows.
_RTL_HEBREW_FONT_SEARCH_PATHS = [
"/usr/share/fonts/truetype/noto/NotoSansHebrew-Regular.ttf",
"/usr/share/fonts/opentype/noto/NotoSansHebrew-Regular.ttf",
"/app/fonts/NotoSansHebrew-Regular.ttf",
"C:/Windows/Fonts/arial.ttf",
]
def __init__(self, provider=None):
self._provider = provider
self._font_path: Optional[str] = None
# script group ("arabic"/"hebrew"/"") → resolved font path, so a
# single instance can serve Arabic and Hebrew jobs in a row.
self._rtl_font_paths: Dict[str, str] = {}
self._translation_stats = {"attempted": 0, "changed": 0}
self._custom_prompt: Optional[str] = None
# OCR overrides (admin settings); None → fall back to config.MISTRAL_*
@@ -184,14 +299,56 @@ class PDFTranslator:
recorder = getattr(self, "_segment_recorder", None)
return recorder.get_pairs() if recorder is not None else []
def _get_font_path(self) -> Optional[str]:
"""Resolve a Unicode-capable TTF/OTF font file."""
if self._font_path is not None:
return self._font_path
for p in self._FONT_SEARCH_PATHS:
@staticmethod
def _first_existing_font(paths) -> Optional[str]:
for p in paths:
if Path(p).exists():
self._font_path = p
return p
return None
def _get_font_path(self, target_language: str = "") -> Optional[str]:
"""Resolve a Unicode-capable TTF/OTF font file.
RTL targets first search a font covering THEIR script — Noto
Naskh/Sans Arabic for the Arabic script (ar, fa, ur…), Noto Sans
Hebrew for Hebrew (he, yi); other RTL scripts (Thaana "dv") use
the generic Unicode list directly. A missing script font falls
back to the generic list with a warning — a missing font
degrades the rendering but never fails the job. Non-RTL targets
use the generic list only (unchanged behavior).
"""
if not is_rtl(target_language):
if self._font_path:
return self._font_path
found = self._first_existing_font(self._FONT_SEARCH_PATHS)
if not found:
logger.warning("no_unicode_font_found")
return None
self._font_path = found
return found
script = _rtl_script(target_language)
cached = self._rtl_font_paths.get(script)
if cached:
return cached
if script == "arabic":
preferred = self._RTL_ARABIC_FONT_SEARCH_PATHS
elif script == "hebrew":
preferred = self._RTL_HEBREW_FONT_SEARCH_PATHS
else:
preferred = []
found = self._first_existing_font(preferred + self._FONT_SEARCH_PATHS)
if found:
if preferred and found not in preferred:
logger.warning(
"rtl_script_font_not_found_current_font_used",
script=script,
font=found,
)
self._rtl_font_paths[script] = found
return found
logger.warning("no_unicode_font_found")
return None
@@ -259,7 +416,7 @@ class PDFTranslator:
doc.close()
raise RuntimeError("PDF has no pages.")
font_path = self._get_font_path()
font_path = self._get_font_path(target_language)
logger.info(
"pdf_layout_start",
pages=total_pages,
@@ -311,7 +468,7 @@ class PDFTranslator:
"""Core PyMuPDF in-place processing — one page at a time."""
import fitz
is_rtl = target_language.lower() in RTL_LANGUAGES
rtl_target = is_rtl(target_language)
total_blocks = 0
translated_blocks = 0
@@ -514,7 +671,7 @@ class PDFTranslator:
for block in blocks:
if block.get("translated"):
self._write_translated_block(
page, block, font_path, is_rtl
page, block, font_path, rtl_target
)
if progress_callback:
@@ -871,7 +1028,7 @@ class PDFTranslator:
page,
block: Dict,
font_path: Optional[str],
is_rtl: bool,
rtl_target: bool,
) -> bool:
"""Write translated text into the block's bounding box.
@@ -897,6 +1054,12 @@ class PDFTranslator:
original_rect = fitz.Rect(block["bbox"])
translated = block["translated"]
if rtl_target:
# Shape Arabic-script text (contextual letter forms + visual
# order) before insertion — insert_textbox does no bidi
# processing of its own. Missing/failed libs degrade to the
# raw text (warning logged inside _shape_rtl).
translated = _shape_rtl(translated)
target_size = block["font_size"]
color = self._int_to_rgb(block["color"])
@@ -1443,8 +1606,20 @@ class PDFTranslator:
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak
from reportlab.lib.styles import getSampleStyleSheet
is_rtl = target_language.lower() in RTL_LANGUAGES
alignment = TA_RIGHT if is_rtl else TA_JUSTIFY
rtl_target = is_rtl(target_language)
alignment = TA_RIGHT if rtl_target else TA_JUSTIFY
# RTL targets need a TTF font covering the target script (the
# built-in Helvetica has no Arabic/Hebrew glyphs). Registration
# failures degrade to Helvetica with a warning — never a failed
# job.
font_name = "Helvetica"
if rtl_target:
rtl_font = self._get_font_path(target_language)
if rtl_font:
font_name = _register_rtl_font(rtl_font)
else:
logger.warning("pdf_rtl_font_missing_helvetica_fallback")
styles = getSampleStyleSheet()
@@ -1464,6 +1639,8 @@ class PDFTranslator:
leading=16,
spaceAfter=6,
alignment=alignment,
fontName=font_name,
wordWrap="RTL" if rtl_target else None,
textColor=colors.HexColor("#1a1a1a"),
)
@@ -1490,6 +1667,11 @@ class PDFTranslator:
elements.append(Spacer(1, 4))
continue
if rtl_target:
# Shape Arabic-script text before rendering —
# reportlab does no bidi processing of its own.
para_text = _shape_rtl(para_text)
safe = (
para_text
.replace("&", "&amp;")