From cae09fc512952fe98924e4e1b756fd63d0519959 Mon Sep 17 00:00:00 2001 From: sepehr Date: Wed, 2 Sep 2026 21:22:22 +0200 Subject: [PATCH] fix(pdf): corriger la mise en page et l'affichage des traductions en persan et RTL - Prioriser les polices couvrant l'alphabet latin et arabo-persan (DejaVu Sans, Arial) afin d'eviter les debordements et caracteres manquants - Decouper et mettre en miroir le texte ligne par ligne pour preserver l'ordre vertical de lecture du haut vers le bas - Ignorer les fonds de diapositives pleine page pour eviter de tronconner les phrases sur fond colore - Supporter les listes avec alinea pour fusionner les phrases longues sans coupure de mots - Ajouter fonts-dejavu-core au conteneur Docker de production --- docker/backend/Dockerfile | 1 + tests/test_translators/test_rtl_layout.py | 42 ++++++ translators/pdf_translator.py | 149 +++++++++++++++++----- 3 files changed, 163 insertions(+), 29 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 7d17277..85a6563 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -31,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libmagic1 \ libpq5 \ curl \ + fonts-dejavu-core \ fonts-noto \ fonts-noto-cjk \ fonts-noto-cjk-extra \ diff --git a/tests/test_translators/test_rtl_layout.py b/tests/test_translators/test_rtl_layout.py index 8afd809..e46ef3d 100644 --- a/tests/test_translators/test_rtl_layout.py +++ b/tests/test_translators/test_rtl_layout.py @@ -1183,3 +1183,45 @@ class TestLtrNormalization: assert "Helvetica" not in fonts, fonts assert "?" not in text assert any(0xFB50 <= ord(c) <= 0xFEFF for c in text) + + @pytest.mark.skipif(_host_rtl_font() is None, reason="no Arabic-capable font on this host") + def test_rtl_multiline_order_preserved(self, tmp_path): + """Multi-line RTL text must wrap lines top-to-bottom so the start + of the paragraph appears on the first line, not upside-down.""" + import fitz + + src = tmp_path / "src_multiline.pdf" + doc = fitz.open() + page = doc.new_page(width=300, height=200) + # Narrow box to force text into multiple lines + page.insert_textbox( + fitz.Rect(50, 50, 250, 150), + "1. Première étape d'installation avec des instructions détaillées.", + fontsize=12, fontname="helv", + ) + doc.save(str(src)) + doc.close() + + fa_text = "1. مرحله اول نصب با دستورالعمل های دقیق و کامل برای ساعت هوشمند." + translator = PDFTranslator( + provider=MockTranslationProvider({ + "1. Première étape d'installation avec des instructions détaillées.": fa_text + }) + ) + out = tmp_path / "out_multiline.pdf" + translator.translate_file(src, out, "fa") + + doc2 = fitz.open(str(out)) + page_dict = doc2[0].get_text("dict") + doc2.close() + + lines = [ + "".join(s["text"] for s in l["spans"]) + for b in page_dict["blocks"] if "lines" in b + for l in b["lines"] + ] + assert len(lines) >= 2 + # The number '1.' and start of text must appear on the FIRST line + assert "1." in lines[0] or "1" in lines[0] + # Must not contain overflow marker + assert "[translation overflow]" not in " ".join(lines) diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 3114048..5c0f13b 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -96,6 +96,87 @@ def _shape_rtl(text: str) -> str: return text +def _shape_rtl_multiline( + text: str, + max_width: float = 0.0, + fontsize: float = 12.0, + fontfile: Optional[str] = None, +) -> str: + """Shape Arabic-script text line by line to preserve top-to-bottom reading. + + When text spans multiple lines, applying get_display globally reverses the + entire paragraph, causing LTR engines (like PyMuPDF) to render lines + from bottom to top. By wrapping words into lines first according to max_width + and shaping each line individually, line 1 stays at the top and the final line + at the bottom. + """ + if not text: + return text + try: + import arabic_reshaper + from bidi.algorithm import get_display + except Exception: + return _shape_rtl(text) + + # If no width provided or width is invalid, shape each line of existing newlines + if max_width <= 0: + paragraphs = text.splitlines() + shaped_paras = [] + for p in paragraphs: + if not p.strip(): + shaped_paras.append(p) + else: + try: + shaped_paras.append(get_display(arabic_reshaper.reshape(p))) + except Exception: + shaped_paras.append(p) + return "\n".join(shaped_paras) + + # Try to load font for measurement + measure_font = None + if fontfile: + try: + import fitz + measure_font = fitz.Font(fontfile=fontfile) + except Exception: + measure_font = None + + shaped_result = [] + for para in text.splitlines(): + if not para.strip(): + shaped_result.append("") + continue + + words = para.split(" ") + lines = [] + cur_words = [] + + if measure_font: + for w in words: + if not w: + continue + cand = (" ".join(cur_words + [w])) if cur_words else w + if measure_font.text_length(cand, fontsize=fontsize) <= max_width or not cur_words: + cur_words.append(w) + else: + lines.append(" ".join(cur_words)) + cur_words = [w] + if cur_words: + lines.append(" ".join(cur_words)) + else: + lines = [para] + + shaped_lines = [] + for l in lines: + try: + shaped_lines.append(get_display(arabic_reshaper.reshape(l))) + except Exception: + shaped_lines.append(l) + shaped_result.append("\n".join(shaped_lines)) + + return "\n".join(shaped_result) + + # Font path → reportlab font name, for RTL fonts registered in this # process. A fixed name would make two consecutive translations with # different fonts (e.g. Arabic then Hebrew) overwrite each other's @@ -221,18 +302,19 @@ class PDFTranslator: ] # Arabic-script-capable fonts (ar, fa, ur, ps, ku, sd, ug, ckb), - # searched FIRST for those targets. Debian's fonts-noto package - # ships Noto Naskh Arabic and Noto Sans Arabic under - # /usr/share/fonts/truetype/noto/; on Windows, Arial covers the - # whole Arabic script. + # searched FIRST for those targets. Prioritize fonts covering BOTH + # Latin and Arabic scripts (DejaVuSans, Arial) with standard line heights, + # so English brands/words and tight slide bboxes render without overflow or tofu. _RTL_ARABIC_FONT_SEARCH_PATHS = [ - "/usr/share/fonts/truetype/noto/NotoNaskhArabic-Regular.ttf", - "/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf", - "/usr/share/fonts/opentype/noto/NotoNaskhArabic-Regular.ttf", - "/usr/share/fonts/opentype/noto/NotoSansArabic-Regular.ttf", - "/app/fonts/NotoNaskhArabic-Regular.ttf", - "/app/fonts/NotoSansArabic-Regular.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/app/fonts/DejaVuSans.ttf", "C:/Windows/Fonts/arial.ttf", + "/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf", + "/usr/share/fonts/opentype/noto/NotoSansArabic-Regular.ttf", + "/app/fonts/NotoSansArabic-Regular.ttf", + "/usr/share/fonts/truetype/noto/NotoNaskhArabic-Regular.ttf", + "/usr/share/fonts/opentype/noto/NotoNaskhArabic-Regular.ttf", + "/app/fonts/NotoNaskhArabic-Regular.ttf", ] # Hebrew-script-capable fonts (he, yi), searched FIRST for those @@ -733,12 +815,17 @@ class PDFTranslator: # "code block" or "callout" patterns and avoid merging their # constituent lines. drawing_rects: list = [] + page_area = (page.rect.width * page.rect.height) if (page.rect.width and page.rect.height) else 1.0 if hasattr(page, "get_drawings"): try: for d in page.get_drawings(): r = d.get("rect") if r is not None and d.get("fill") is not None: - drawing_rects.append(fitz.Rect(r)) + dr = fitz.Rect(r) + # Exclude full-page or card backgrounds (>= 35% of page area) + # so that slide backgrounds do not break normal paragraphs into lines. + if (dr.width * dr.height) / page_area < 0.35: + drawing_rects.append(dr) except Exception: pass @@ -959,14 +1046,18 @@ class PDFTranslator: if vertical_gap < 0 or vertical_gap > line_height * 1.5: return False - # Similar horizontal position (within 15pt) - if abs(a_bbox[0] - b_bbox[0]) > 15: + # Horizontal alignment: left-aligned (within 15pt) OR hanging indent (line b indented up to 36pt) + x_diff = abs(a_bbox[0] - b_bbox[0]) + is_hanging_indent = (0 <= b_bbox[0] - a_bbox[0] <= 36.0) + if x_diff > 15 and not is_hanging_indent: return False - # Don't merge if widths are very different (likely different columns) + # If b is shorter than a, it is typically the last line of the paragraph, + # provided it does not extend significantly past a's right edge. + # Only reject if b extends far past a's right margin. a_width = a_bbox[2] - a_bbox[0] b_width = b_bbox[2] - b_bbox[0] - if a_width > 0 and abs(b_width - a_width) / a_width > 0.5: + if b_bbox[2] > a_bbox[2] + 25 and b_width > a_width * 1.5: return False return True @@ -1054,12 +1145,6 @@ class PDFTranslator: original_rect = fitz.Rect(block["bbox"]) translated = block["translated"] - if rtl_target: - # Shape Arabic-script text (contextual letter forms + visual - # order) before insertion — insert_textbox does no bidi - # processing of its own. Missing/failed libs degrade to the - # raw text (warning logged inside _shape_rtl). - translated = _shape_rtl(translated) target_size = block["font_size"] color = self._int_to_rgb(block["color"]) @@ -1126,38 +1211,38 @@ class PDFTranslator: # Tier 0: original rect, original size # insert_textbox returns >= 0 if text fit, < 0 if overflow (in points) - rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align) + rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True # Tier 1: expanded horizontal, original size if expanded_h.width > original_rect.width + 1: - rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align) + rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True # Tier 2: expanded vertical, original size if expanded_v.height > original_rect.height + 1: - rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align) + rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True # Tier 3: shrink once size_1 = max(target_size * FONT_SHRINK_FACTOR, min_size) if size_1 < target_size - 0.1: # only try if shrink is meaningful - rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align) + rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True # Tier 4: shrink twice size_2 = max(target_size * FONT_SHRINK_FACTOR * FONT_SHRINK_FACTOR, min_size) if size_2 < size_1 - 0.1: - rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align) + rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True # Tier 5: hit the min size floor - rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align) + rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align, rtl_target=rtl_target) if rc is not None and rc >= 0: return True @@ -1190,13 +1275,19 @@ class PDFTranslator: pass def _try_insert( - self, page, rect, text, fontsize, fontname, fontfile, color, align + self, page, rect, text, fontsize, fontname, fontfile, color, align, rtl_target: bool = False ): """Attempt insert_textbox, returns rc or None on error.""" try: + if rtl_target and text: + text_to_insert = _shape_rtl_multiline( + text, max_width=rect.width, fontsize=fontsize, fontfile=fontfile + ) + else: + text_to_insert = text return page.insert_textbox( rect, - text, + text_to_insert, fontsize=fontsize, fontname=fontname, fontfile=fontfile,