feat(rtl): rendu droite-a-gauche complet pour Word, PowerPoint, Excel et PDF
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m37s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m37s
- source unique RTL_LANGUAGES/is_rtl dans core/languages.py (fin des 3 copies) - Word: bidi partout (corps, tableaux bidiVisual, notes/fin/commentaires, zones de texte, 6 zones d'en-tetes/pieds), insertion OOXML ordonnee, polices cs elargies aux 11 langues - PowerPoint: alignements explicites preserves, notes du presentateur, indice de police <a:cs> insert a sa place - Excel: feuilles affichees de droite a gauche, feuilles graphiques ignorees - PDF: faconnage bidi (arabic-reshaper + python-bidi), polices par ecriture (arabe/hebreu), TTF enregistree pour le PDF recompose - 46 tests nouveaux (tests/test_translators/test_rtl_layout.py), 253 au total
This commit is contained in:
@@ -33,10 +33,12 @@ Scanned PDFs:
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Callable, List
|
||||
|
||||
from core.logging import get_logger
|
||||
from core.languages import is_rtl
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -60,8 +62,70 @@ HEADING_MIN_SCALE = 0.90
|
||||
# Maximum font shrink for body text
|
||||
BODY_MIN_SCALE = 0.75
|
||||
|
||||
# RTL language codes
|
||||
RTL_LANGUAGES = frozenset({"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"})
|
||||
# Warn at most once when the RTL shaping libraries are missing.
|
||||
_RTL_SHAPING_MISSING_WARNED = False
|
||||
|
||||
|
||||
def _shape_rtl(text: str) -> str:
|
||||
"""Shape Arabic-script text for PDF engines that do no bidi of their own.
|
||||
|
||||
arabic-reshaper joins the letters into their contextual forms and
|
||||
python-bidi reorders the string for visual (left-to-right) rendering.
|
||||
Both libraries are optional on purpose: when they are missing or
|
||||
fail, the text is returned unchanged with a warning — the
|
||||
translation itself must always succeed, only the rendering degrades.
|
||||
"""
|
||||
global _RTL_SHAPING_MISSING_WARNED
|
||||
if not text:
|
||||
return text
|
||||
try:
|
||||
import arabic_reshaper
|
||||
from bidi.algorithm import get_display
|
||||
except Exception:
|
||||
if not _RTL_SHAPING_MISSING_WARNED:
|
||||
logger.warning(
|
||||
"rtl_shaping_libs_missing",
|
||||
hint="pip install arabic-reshaper python-bidi for correct RTL rendering",
|
||||
)
|
||||
_RTL_SHAPING_MISSING_WARNED = True
|
||||
return text
|
||||
try:
|
||||
return get_display(arabic_reshaper.reshape(text))
|
||||
except Exception as e:
|
||||
logger.warning("rtl_shaping_failed", error=str(e)[:200])
|
||||
return text
|
||||
|
||||
|
||||
# Font path → reportlab font name, for RTL fonts registered in this
|
||||
# process. A fixed name would make two consecutive translations with
|
||||
# different fonts (e.g. Arabic then Hebrew) overwrite each other's
|
||||
# registration.
|
||||
_RTL_FONT_REGISTRATIONS: Dict[str, str] = {}
|
||||
|
||||
|
||||
def _register_rtl_font(font_path: str) -> str:
|
||||
"""Register a TTF under a stable per-path name, once per process.
|
||||
|
||||
Returns the reportlab font name, or "Helvetica" when registration
|
||||
fails (warning logged — a font problem must never fail the job).
|
||||
"""
|
||||
registered = _RTL_FONT_REGISTRATIONS.get(font_path)
|
||||
if registered:
|
||||
return registered
|
||||
try:
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
|
||||
digest = format(zlib.crc32(str(font_path).encode("utf-8")) & 0xFFFFFFFF, "08x")
|
||||
name = f"WordlyRTL-{digest}"
|
||||
pdfmetrics.registerFont(TTFont(name, font_path))
|
||||
_RTL_FONT_REGISTRATIONS[font_path] = name
|
||||
return name
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"pdf_rtl_font_register_failed", error=str(e)[:200], font=font_path
|
||||
)
|
||||
return "Helvetica"
|
||||
|
||||
|
||||
def _record_format_loss_metric(element_type: str, count: int = 1) -> None:
|
||||
@@ -109,6 +173,29 @@ def _libreoffice_available() -> bool:
|
||||
_libreoffice_available._cache = None # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# Script groups among the RTL languages — the preferred font depends on
|
||||
# the writing system, not just the direction: a Hebrew target would
|
||||
# render as tofu with an Arabic-only font.
|
||||
_ARABIC_SCRIPT_LANGUAGES = frozenset(
|
||||
{"ar", "fa", "ur", "ps", "ku", "sd", "ug", "ckb"}
|
||||
)
|
||||
_HEBREW_SCRIPT_LANGUAGES = frozenset({"he", "yi"})
|
||||
|
||||
|
||||
def _rtl_script(code: str) -> str:
|
||||
"""Script group of an RTL language code: "arabic", "hebrew" or ""
|
||||
|
||||
("" meaning: use the generic Unicode font list — e.g. Thaana "dv",
|
||||
for which no dedicated font is searched).
|
||||
"""
|
||||
base = (code or "").strip().replace("_", "-").split("-")[0].lower()
|
||||
if base in _ARABIC_SCRIPT_LANGUAGES:
|
||||
return "arabic"
|
||||
if base in _HEBREW_SCRIPT_LANGUAGES:
|
||||
return "hebrew"
|
||||
return ""
|
||||
|
||||
|
||||
class PDFTranslator:
|
||||
"""Translates PDF files with layout preservation using PyMuPDF."""
|
||||
|
||||
@@ -133,9 +220,37 @@ class PDFTranslator:
|
||||
"/System/Library/Fonts/Helvetica.ttc",
|
||||
]
|
||||
|
||||
# Arabic-script-capable fonts (ar, fa, ur, ps, ku, sd, ug, ckb),
|
||||
# searched FIRST for those targets. Debian's fonts-noto package
|
||||
# ships Noto Naskh Arabic and Noto Sans Arabic under
|
||||
# /usr/share/fonts/truetype/noto/; on Windows, Arial covers the
|
||||
# whole Arabic script.
|
||||
_RTL_ARABIC_FONT_SEARCH_PATHS = [
|
||||
"/usr/share/fonts/truetype/noto/NotoNaskhArabic-Regular.ttf",
|
||||
"/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf",
|
||||
"/usr/share/fonts/opentype/noto/NotoNaskhArabic-Regular.ttf",
|
||||
"/usr/share/fonts/opentype/noto/NotoSansArabic-Regular.ttf",
|
||||
"/app/fonts/NotoNaskhArabic-Regular.ttf",
|
||||
"/app/fonts/NotoSansArabic-Regular.ttf",
|
||||
"C:/Windows/Fonts/arial.ttf",
|
||||
]
|
||||
|
||||
# Hebrew-script-capable fonts (he, yi), searched FIRST for those
|
||||
# targets. Same Debian package provides Noto Sans Hebrew; Arial
|
||||
# covers Hebrew on Windows.
|
||||
_RTL_HEBREW_FONT_SEARCH_PATHS = [
|
||||
"/usr/share/fonts/truetype/noto/NotoSansHebrew-Regular.ttf",
|
||||
"/usr/share/fonts/opentype/noto/NotoSansHebrew-Regular.ttf",
|
||||
"/app/fonts/NotoSansHebrew-Regular.ttf",
|
||||
"C:/Windows/Fonts/arial.ttf",
|
||||
]
|
||||
|
||||
def __init__(self, provider=None):
|
||||
self._provider = provider
|
||||
self._font_path: Optional[str] = None
|
||||
# script group ("arabic"/"hebrew"/"") → resolved font path, so a
|
||||
# single instance can serve Arabic and Hebrew jobs in a row.
|
||||
self._rtl_font_paths: Dict[str, str] = {}
|
||||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||||
self._custom_prompt: Optional[str] = None
|
||||
# OCR overrides (admin settings); None → fall back to config.MISTRAL_*
|
||||
@@ -184,14 +299,56 @@ class PDFTranslator:
|
||||
recorder = getattr(self, "_segment_recorder", None)
|
||||
return recorder.get_pairs() if recorder is not None else []
|
||||
|
||||
def _get_font_path(self) -> Optional[str]:
|
||||
"""Resolve a Unicode-capable TTF/OTF font file."""
|
||||
if self._font_path is not None:
|
||||
return self._font_path
|
||||
for p in self._FONT_SEARCH_PATHS:
|
||||
@staticmethod
|
||||
def _first_existing_font(paths) -> Optional[str]:
|
||||
for p in paths:
|
||||
if Path(p).exists():
|
||||
self._font_path = p
|
||||
return p
|
||||
return None
|
||||
|
||||
def _get_font_path(self, target_language: str = "") -> Optional[str]:
|
||||
"""Resolve a Unicode-capable TTF/OTF font file.
|
||||
|
||||
RTL targets first search a font covering THEIR script — Noto
|
||||
Naskh/Sans Arabic for the Arabic script (ar, fa, ur…), Noto Sans
|
||||
Hebrew for Hebrew (he, yi); other RTL scripts (Thaana "dv") use
|
||||
the generic Unicode list directly. A missing script font falls
|
||||
back to the generic list with a warning — a missing font
|
||||
degrades the rendering but never fails the job. Non-RTL targets
|
||||
use the generic list only (unchanged behavior).
|
||||
"""
|
||||
if not is_rtl(target_language):
|
||||
if self._font_path:
|
||||
return self._font_path
|
||||
found = self._first_existing_font(self._FONT_SEARCH_PATHS)
|
||||
if not found:
|
||||
logger.warning("no_unicode_font_found")
|
||||
return None
|
||||
self._font_path = found
|
||||
return found
|
||||
|
||||
script = _rtl_script(target_language)
|
||||
cached = self._rtl_font_paths.get(script)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
if script == "arabic":
|
||||
preferred = self._RTL_ARABIC_FONT_SEARCH_PATHS
|
||||
elif script == "hebrew":
|
||||
preferred = self._RTL_HEBREW_FONT_SEARCH_PATHS
|
||||
else:
|
||||
preferred = []
|
||||
|
||||
found = self._first_existing_font(preferred + self._FONT_SEARCH_PATHS)
|
||||
if found:
|
||||
if preferred and found not in preferred:
|
||||
logger.warning(
|
||||
"rtl_script_font_not_found_current_font_used",
|
||||
script=script,
|
||||
font=found,
|
||||
)
|
||||
self._rtl_font_paths[script] = found
|
||||
return found
|
||||
logger.warning("no_unicode_font_found")
|
||||
return None
|
||||
|
||||
@@ -259,7 +416,7 @@ class PDFTranslator:
|
||||
doc.close()
|
||||
raise RuntimeError("PDF has no pages.")
|
||||
|
||||
font_path = self._get_font_path()
|
||||
font_path = self._get_font_path(target_language)
|
||||
logger.info(
|
||||
"pdf_layout_start",
|
||||
pages=total_pages,
|
||||
@@ -311,7 +468,7 @@ class PDFTranslator:
|
||||
"""Core PyMuPDF in-place processing — one page at a time."""
|
||||
import fitz
|
||||
|
||||
is_rtl = target_language.lower() in RTL_LANGUAGES
|
||||
rtl_target = is_rtl(target_language)
|
||||
total_blocks = 0
|
||||
translated_blocks = 0
|
||||
|
||||
@@ -514,7 +671,7 @@ class PDFTranslator:
|
||||
for block in blocks:
|
||||
if block.get("translated"):
|
||||
self._write_translated_block(
|
||||
page, block, font_path, is_rtl
|
||||
page, block, font_path, rtl_target
|
||||
)
|
||||
|
||||
if progress_callback:
|
||||
@@ -871,7 +1028,7 @@ class PDFTranslator:
|
||||
page,
|
||||
block: Dict,
|
||||
font_path: Optional[str],
|
||||
is_rtl: bool,
|
||||
rtl_target: bool,
|
||||
) -> bool:
|
||||
"""Write translated text into the block's bounding box.
|
||||
|
||||
@@ -897,6 +1054,12 @@ class PDFTranslator:
|
||||
|
||||
original_rect = fitz.Rect(block["bbox"])
|
||||
translated = block["translated"]
|
||||
if rtl_target:
|
||||
# Shape Arabic-script text (contextual letter forms + visual
|
||||
# order) before insertion — insert_textbox does no bidi
|
||||
# processing of its own. Missing/failed libs degrade to the
|
||||
# raw text (warning logged inside _shape_rtl).
|
||||
translated = _shape_rtl(translated)
|
||||
target_size = block["font_size"]
|
||||
|
||||
color = self._int_to_rgb(block["color"])
|
||||
@@ -1443,8 +1606,20 @@ class PDFTranslator:
|
||||
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak
|
||||
from reportlab.lib.styles import getSampleStyleSheet
|
||||
|
||||
is_rtl = target_language.lower() in RTL_LANGUAGES
|
||||
alignment = TA_RIGHT if is_rtl else TA_JUSTIFY
|
||||
rtl_target = is_rtl(target_language)
|
||||
alignment = TA_RIGHT if rtl_target else TA_JUSTIFY
|
||||
|
||||
# RTL targets need a TTF font covering the target script (the
|
||||
# built-in Helvetica has no Arabic/Hebrew glyphs). Registration
|
||||
# failures degrade to Helvetica with a warning — never a failed
|
||||
# job.
|
||||
font_name = "Helvetica"
|
||||
if rtl_target:
|
||||
rtl_font = self._get_font_path(target_language)
|
||||
if rtl_font:
|
||||
font_name = _register_rtl_font(rtl_font)
|
||||
else:
|
||||
logger.warning("pdf_rtl_font_missing_helvetica_fallback")
|
||||
|
||||
styles = getSampleStyleSheet()
|
||||
|
||||
@@ -1464,6 +1639,8 @@ class PDFTranslator:
|
||||
leading=16,
|
||||
spaceAfter=6,
|
||||
alignment=alignment,
|
||||
fontName=font_name,
|
||||
wordWrap="RTL" if rtl_target else None,
|
||||
textColor=colors.HexColor("#1a1a1a"),
|
||||
)
|
||||
|
||||
@@ -1490,6 +1667,11 @@ class PDFTranslator:
|
||||
elements.append(Spacer(1, 4))
|
||||
continue
|
||||
|
||||
if rtl_target:
|
||||
# Shape Arabic-script text before rendering —
|
||||
# reportlab does no bidi processing of its own.
|
||||
para_text = _shape_rtl(para_text)
|
||||
|
||||
safe = (
|
||||
para_text
|
||||
.replace("&", "&")
|
||||
|
||||
Reference in New Issue
Block a user