diff --git a/config.py b/config.py index c405080..5a8d26a 100644 --- a/config.py +++ b/config.py @@ -104,6 +104,19 @@ class Config: # When true, only Pro+ plans can use L2. Otherwise, all plans can. QUALITY_L2_TIER_GATE = os.getenv("QUALITY_L2_TIER_GATE", "true").lower() == "true" + # ============== PDF Smart-Fit (Track B3.5) ============== + # When true, the PDF translator uses a smart overflow strategy: + # 1. Try original bbox at original size + # 2. Expand bbox vertically (3x original height) + # 3. Shrink font ONCE (0.93x) with expanded bbox + # 4. Shrink font AGAIN (0.87x cumulative) with expanded bbox + # 5. For headings (font >= 14pt): never below 90% of original + # 6. For body: never below 75% of original + # 7. If still overflow: skip block, log format_loss, write placeholder + # + # Set to false to use the legacy aggressive-shrink strategy (NOT recommended). + PDF_SMART_FIT_ENABLED = os.getenv("PDF_SMART_FIT_ENABLED", "true").lower() == "true" + # ============== API Configuration ============== API_TITLE = "Document Translation API" diff --git a/sample_files/test_corpus/test_pdf_translated.pdf b/sample_files/test_corpus/test_pdf_translated.pdf new file mode 100644 index 0000000..2a0aad2 Binary files /dev/null and b/sample_files/test_corpus/test_pdf_translated.pdf differ diff --git a/scripts/compare_pdfs.py b/scripts/compare_pdfs.py new file mode 100644 index 0000000..cc5c59b --- /dev/null +++ b/scripts/compare_pdfs.py @@ -0,0 +1,107 @@ +""" +Deep comparison of original vs translated PDF. + +Outputs a detailed diff to identify exactly what format elements were lost. +""" +import fitz +from pathlib import Path +import sys + +original = Path("sample_files/test_corpus/test_pdf.pdf") +translated = Path("sample_files/test_corpus/test_pdf_translated.pdf") + +print(f"Original: {original.stat().st_size:>10} bytes") +print(f"Translated: {translated.stat().st_size:>10} bytes") +print(f"Size diff: {translated.stat().st_size - original.stat().st_size:>10} bytes ({100 * translated.stat().st_size / original.stat().st_size:.1f}%)") +print() + +doc_orig = fitz.open(str(original)) +doc_trans = fitz.open(str(translated)) + +print("=" * 70) +print("PAGE COUNT") +print("=" * 70) +print(f" Original: {len(doc_orig)} pages") +print(f" Translated: {len(doc_trans)} pages") +print() + +print("=" * 70) +print("PER-PAGE COMPARISON") +print("=" * 70) +for i in range(max(len(doc_orig), len(doc_trans))): + o_page = doc_orig[i] if i < len(doc_orig) else None + t_page = doc_trans[i] if i < len(doc_trans) else None + print(f"\n--- Page {i+1} ---") + if o_page: + o_text = len(o_page.get_text()) + o_links = len(o_page.get_links()) + o_images = len(o_page.get_images()) + o_drawings = len(o_page.get_drawings()) + print(f" Original: {o_text:>4} chars, {o_links:>2} links, {o_images:>2} images, {o_drawings:>3} drawings") + if t_page: + t_text = len(t_page.get_text()) + t_links = len(t_page.get_links()) + t_images = len(t_page.get_images()) + t_drawings = len(t_page.get_drawings()) + print(f" Translated: {t_text:>4} chars, {t_links:>2} links, {t_images:>2} images, {t_drawings:>3} drawings") + if o_page and t_page: + print(f" DIFF: text {t_text - o_text:+d}, links {t_links - o_links:+d}, images {t_images - o_images:+d}, drawings {t_drawings - o_drawings:+d}") + +print() +print("=" * 70) +print("HYPERLINK COMPARISON") +print("=" * 70) +for i in range(max(len(doc_orig), len(doc_trans))): + o_page = doc_orig[i] if i < len(doc_orig) else None + t_page = doc_trans[i] if i < len(doc_trans) else None + o_links = o_page.get_links() if o_page else [] + t_links = t_page.get_links() if t_page else [] + if o_links or t_links: + print(f"\n Page {i+1}:") + print(f" Original has {len(o_links)} links:") + for link in o_links: + kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?" + uri_or_page = link.get("uri") or f"page {link.get('page')}" + print(f" - {kind}: {uri_or_page[:80]}") + print(f" Translated has {len(t_links)} links:") + for link in t_links: + kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?" + uri_or_page = link.get("uri") or f"page {link.get('page')}" + print(f" - {kind}: {uri_or_page[:80]}") + +print() +print("=" * 70) +print("IMAGE COMPARISON") +print("=" * 70) +total_o_imgs = sum(len(p.get_images()) for p in doc_orig) +total_t_imgs = sum(len(p.get_images()) for p in doc_trans) +print(f" Original: {total_o_imgs} images") +print(f" Translated: {total_t_imgs} images") +if total_o_imgs > 0 and total_t_imgs == 0: + print(" ❌ ALL IMAGES LOST in translation!") + +print() +print("=" * 70) +print("DRAWING COMPARISON (rects, lines, etc.)") +print("=" * 70) +total_o_dr = sum(len(p.get_drawings()) for p in doc_orig) +total_t_dr = sum(len(p.get_drawings()) for p in doc_trans) +print(f" Original: {total_o_dr} drawings") +print(f" Translated: {total_t_dr} drawings") + +print() +print("=" * 70) +print("SAMPLE TEXT — Page 1 of translated") +print("=" * 70) +if len(doc_trans) > 0: + print(doc_trans[0].get_text()[:500]) + +print() +print("=" * 70) +print("SAMPLE TEXT — Page 1 of original") +print("=" * 70) +if len(doc_orig) > 0: + print(doc_orig[0].get_text()[:500]) + +doc_orig.close() +doc_trans.close() diff --git a/scripts/debug_rc_real.py b/scripts/debug_rc_real.py new file mode 100644 index 0000000..1c5905f --- /dev/null +++ b/scripts/debug_rc_real.py @@ -0,0 +1,40 @@ +"""Debug: show actual rc values for the title block in real translation.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Patch _try_insert to log rc values +original_try_insert = pdf_mod.PDFTranslator._try_insert +TITLES_TO_TRACE = {"Spécification technique : Office Translator v3.0", "Technical Specification: Office Translator v3.0"} + +def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align): + rc = page.insert_textbox( + rect, text, + fontsize=fontsize, + fontname=fontname, + fontfile=fontfile, + color=color, + align=align, + overlay=True, + ) + # Only trace the title + is_title = any(t in text for t in ["Spécification technique", "Technical Specification"]) + if is_title: + print(f" [TITLE] size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} -> rc={rc}", flush=True) + return rc + +pdf_mod.PDFTranslator._try_insert = debug_try_insert + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/debug_rc_values.py b/scripts/debug_rc_values.py new file mode 100644 index 0000000..8422b69 --- /dev/null +++ b/scripts/debug_rc_values.py @@ -0,0 +1,37 @@ +"""Debug the actual rc values returned by insert_textbox during real translation.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Monkey-patch _try_insert to print rc values +original_try_insert = pdf_mod.PDFTranslator._try_insert + +def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align): + rc = page.insert_textbox( + rect, text, + fontsize=fontsize, + fontname=fontname, + fontfile=fontfile, + color=color, + align=align, + overlay=True, + ) + text_preview = text[:30].replace('\n', '\\n') + print(f" TRY: size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} text={text_preview!r} -> rc={rc}", flush=True) + return rc + +pdf_mod.PDFTranslator._try_insert = debug_try_insert + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/debug_smart_fit.py b/scripts/debug_smart_fit.py new file mode 100644 index 0000000..06319f4 --- /dev/null +++ b/scripts/debug_smart_fit.py @@ -0,0 +1,81 @@ +"""Debug the smart-fit behavior on a real PDF.""" +import sys +sys.path.insert(0, '.') +import importlib.util + +spec = importlib.util.spec_from_file_location("pdf_mod", "translators/pdf_translator.py") +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +import fitz + +# Open the test PDF +doc = fitz.open("sample_files/test_corpus/test_pdf.pdf") +page = doc[0] + +# Extract blocks +translator = pdf_mod.PDFTranslator() +raw_blocks = translator._extract_text_blocks(page) +merged = translator._merge_adjacent_blocks(raw_blocks, page.rect) + +# Show block info +for i, block in enumerate(merged[:5]): + print(f"Block {i}:") + print(f" bbox: {block['bbox']}") + print(f" font_size: {block['font_size']}") + print(f" text: {block['text'][:50]!r}") + print() + +# Manually try the smart-fit on the title block +title = merged[0] +print("=" * 60) +print("Smart-fit trace on title block:") +print("=" * 60) +import fitz +original_rect = fitz.Rect(title["bbox"]) +page_rect = page.rect +print(f" original_rect: {original_rect}") +print(f" page_rect: {page_rect}") +print(f" page width: {page_rect.x1 - page_rect.x0}") +print(f" original width: {original_rect.width}, height: {original_rect.height}") + +# Simulate the expansion +margin = 18 +max_x1 = page_rect.x1 - margin +expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + max_x1, + original_rect.y1, +) +print(f" expanded_h: {expanded_h}, width: {expanded_h.width}") + +next_block_y = page_rect.y1 - margin +max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * pdf_mod.MAX_VERTICAL_EXPANSION, +) +expanded_v = fitz.Rect( + expanded_h.x0, + expanded_h.y0, + expanded_h.x1, + expanded_h.y1 + max_expand_y, +) +print(f" expanded_v: {expanded_v}, width: {expanded_v.width}, height: {expanded_v.height}") + +# Try the actual insert +from fitz import TEXT_ALIGN_LEFT +french_text = "Spécification technique : Office Translator v3.0" +rc = page.insert_textbox( + expanded_v, + french_text, + fontsize=title["font_size"], + fontname="helv", + color=(0.1, 0.2, 0.5), + align=TEXT_ALIGN_LEFT, + overlay=True, +) +print(f" insert_textbox rc (with expanded_v at original size): {rc}") +print(f" >= 0 = fit, < 0 = overflow of {rc} points") + +doc.close() diff --git a/scripts/debug_smart_fit_trace.py b/scripts/debug_smart_fit_trace.py new file mode 100644 index 0000000..c2daa35 --- /dev/null +++ b/scripts/debug_smart_fit_trace.py @@ -0,0 +1,55 @@ +"""Debug trace the smart-fit behavior on a real PDF.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +original_write = pdf_mod.PDFTranslator._write_translated_block + +def debug_write(self, page, block, font_path, is_rtl): + target_size = block['font_size'] + original_rect = fitz.Rect(block['bbox']) + page_rect = page.rect + margin = 18 + is_heading = target_size >= 14 + if is_heading: + expanded_h = original_rect + else: + expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + page_rect.x1 - margin, + original_rect.y1, + ) + next_block_y = page_rect.y1 - margin + max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * 3.0, + ) + expanded_v = fitz.Rect( + expanded_h.x0, + expanded_h.y0, + expanded_h.x1, + expanded_h.y1 + max_expand_y, + ) + text = block['translated'][:40] + is_heading_str = "HEAD" if is_heading else "BODY" + print(f" [{is_heading_str}] text={text!r} | orig={original_rect.width:.0f}x{original_rect.height:.0f} | exp_v={expanded_v.width:.0f}x{expanded_v.height:.0f} | size={target_size}", flush=True) + result = original_write(self, page, block, font_path, is_rtl) + print(f" result: {result}", flush=True) + return result + +pdf_mod.PDFTranslator._write_translated_block = debug_write + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/diagnose_pdf_layout.py b/scripts/diagnose_pdf_layout.py new file mode 100644 index 0000000..f4b25a9 --- /dev/null +++ b/scripts/diagnose_pdf_layout.py @@ -0,0 +1,71 @@ +""" +Deeper investigation: where exactly is the text on each page? + +Compares bounding boxes of the original text vs translated text. +If they're in totally different positions, the layout is broken. +""" +import fitz + +orig = fitz.open("sample_files/test_corpus/test_pdf.pdf") +trans = fitz.open("sample_files/test_corpus/test_pdf_translated.pdf") + +for i in range(min(3, len(orig), len(trans))): + o = orig[i] + t = trans[i] + print(f"\n{'=' * 70}") + print(f"PAGE {i+1}") + print('=' * 70) + + # Get text blocks with positions for original + print(f"\n--- Original (page {i+1}) text blocks ---") + o_blocks = o.get_text("blocks") + for j, b in enumerate(o_blocks[:10]): + x0, y0, x1, y1, text, *_ = b + text_preview = text[:50].replace("\n", " ") + print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'") + + print(f"\n--- Translated (page {i+1}) text blocks ---") + t_blocks = t.get_text("blocks") + for j, b in enumerate(t_blocks[:10]): + x0, y0, x1, y1, text, *_ = b + text_preview = text[:50].replace("\n", " ") + print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'") + + # Check fonts + print(f"\n--- Fonts (page {i+1}) ---") + o_fonts = set() + t_fonts = set() + for blk in o_blocks: + for line in blk[4].split("\n") if blk[4] else []: + pass + # Get fonts from the dict format + o_dict = o.get_text("dict") + t_dict = t.get_text("dict") + for block in o_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}") + for block in t_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}") + print(f" Original fonts: {sorted(o_fonts)[:5]}") + print(f" Translated fonts: {sorted(t_fonts)[:5]}") + + # Check colors used + print(f"\n--- Colors (page {i+1}) ---") + o_colors = set() + t_colors = set() + for block in o_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_colors.add(span.get("color", 0)) + for block in t_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_colors.add(span.get("color", 0)) + print(f" Original colors: {sorted(o_colors)[:5]}") + print(f" Translated colors: {sorted(t_colors)[:5]}") + +orig.close() +trans.close() diff --git a/scripts/rerun_translation.py b/scripts/rerun_translation.py new file mode 100644 index 0000000..2e55773 --- /dev/null +++ b/scripts/rerun_translation.py @@ -0,0 +1,33 @@ +"""Re-run translation with the new B3.5 code and save to disk.""" +import sys +sys.path.insert(0, '.') +import shutil +from pathlib import Path +import fitz + +src = Path('sample_files/test_corpus/test_pdf.pdf') +dst = Path('sample_files/test_corpus/test_pdf_translated.pdf') + +from translators.pdf_translator import PDFTranslator +PDFTranslator().translate_file(src, dst, target_language='fr', source_language='en') + +o = fitz.open(src) +n = fitz.open(dst) +print(f'Original: {len(o)} pages, {src.stat().st_size} bytes') +print(f'Translated: {len(n)} pages, {dst.stat().st_size} bytes') +print() +print('Page 1 font sizes:') +o_sizes = set() +n_sizes = set() +for b in o[0].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + o_sizes.add(round(s.get("size", 0), 1)) +for b in n[0].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + n_sizes.add(round(s.get("size", 0), 1)) +print(' Original: ', sorted(o_sizes)) +print(' Translated:', sorted(n_sizes)) +o.close() +n.close() diff --git a/scripts/test_full_pipeline.py b/scripts/test_full_pipeline.py new file mode 100644 index 0000000..f3bb31c --- /dev/null +++ b/scripts/test_full_pipeline.py @@ -0,0 +1,72 @@ +"""Test if the full pipeline (extract, redact, re-insert links) affects insert_textbox.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Step 1: Extract blocks +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +title = raw_blocks[0] +print(f"Title bbox: {title['bbox']}") + +# Step 2: Simulate redaction +title_rect = fitz.Rect(title['bbox']) +page.add_redact_annot(title_rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) +print("After redaction") + +# Step 3: Simulate link reinsertion (we have TOC links) +links = list(page.get_links()) +print(f"Got {len(links)} links") +for link in links: + from_rect = link.get("from") + if link.get("page") is not None: + page.insert_link({ + "kind": fitz.LINK_GOTO, + "from": from_rect, + "page": link["page"], + "to": fitz.Point(72, 72), + }) + elif link.get("uri"): + page.insert_link({ + "kind": fitz.LINK_URI, + "from": from_rect, + "uri": link["uri"], + }) +print("After link reinsertion") + +# Step 4: NOW try insert_textbox +french = "Spécification technique : Office Translator v3.0" +rect = fitz.Rect(title['bbox']) +expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3) + +# Test on original bbox +rc1 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After full pipeline, original rect: rc={rc1}") + +# Test on expanded_v +rc2 = page.insert_textbox( + expanded_v, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After full pipeline, expanded_v: rc={rc2}") + +doc.close() diff --git a/scripts/test_merged_block.py b/scripts/test_merged_block.py new file mode 100644 index 0000000..8767e99 --- /dev/null +++ b/scripts/test_merged_block.py @@ -0,0 +1,87 @@ +"""Test if merged blocks (TOC) fit in expanded_v.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Extract and merge +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +merged = pdf_mod.PDFTranslator()._merge_adjacent_blocks(raw_blocks, page.rect) + +# Find the TOC block (block 3 in our trace) +toc = merged[3] +print(f"TOC bbox: {toc['bbox']}") +print(f"TOC font_size: {toc['font_size']}") +print(f"TOC text length: {len(toc['text'])}") +print(f"TOC text preview: {toc['text'][:80]!r}") + +# Redact +toc_rect = fitz.Rect(toc['bbox']) +page.add_redact_annot(toc_rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) + +# Test insert with expanded_v +margin = 18 +page_rect = page.rect +expanded_h = fitz.Rect( + max(toc_rect.x0, page_rect.x0 + margin), + toc_rect.y0, + page_rect.x1 - margin, + toc_rect.y1, +) +max_expand_y = min( + page_rect.y1 - margin - toc_rect.y1, + toc_rect.height * 3.0, +) +expanded_v = fitz.Rect( + expanded_h.x0, expanded_h.y0, expanded_h.x1, + expanded_h.y1 + max_expand_y, +) +print(f"expanded_v: {expanded_v.width}x{expanded_v.height}") + +# Translate the TOC text +french_toc = """1. Introduction +.......... 1 +2. Installation et configuration +.......... 2 +3. Moteur de traduction +.......... 3 +4. Préservation des formats +.......... 4 +5. Assurance qualité +.......... 5 +6. Performances et mise à l'échelle +.......... 6 +7. Référence API +.......... 7 +8. Dépannage +.......... 8""" + +# Test on original +rc1 = page.insert_textbox( + toc_rect, french_toc, + fontsize=12, + fontfile='C:/Windows/Fonts/arial.ttf', + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"Original rect: rc={rc1}") + +# Test on expanded_v +rc2 = page.insert_textbox( + expanded_v, french_toc, + fontsize=12, + fontfile='C:/Windows/Fonts/arial.ttf', + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"expanded_v: rc={rc2}") + +doc.close() diff --git a/scripts/test_smart_fit_no_translation.py b/scripts/test_smart_fit_no_translation.py new file mode 100644 index 0000000..2d3e758 --- /dev/null +++ b/scripts/test_smart_fit_no_translation.py @@ -0,0 +1,46 @@ +"""Test smart-fit with mock translation (same length) to isolate the issue.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Mock the translation to return the original text (no length change) +def mock_translate(self, text, target_lang, source_lang): + return text # No change + +PDFTranslator = pdf_mod.PDFTranslator +with patch.object(PDFTranslator, '_translate_single', mock_translate): + PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_no_translation.pdf'), + target_language='fr', + source_language='en', + ) + +# Check fonts +o = fitz.open('sample_files/test_corpus/test_pdf.pdf') +n = fitz.open('sample_files/test_corpus/test_pdf_no_translation.pdf') +print(f"=== Mock translation (no length change) ===") +print(f"Original: {len(o)} pages, {o.page_count} blocks") +print(f"Translated: {len(n)} pages") +for i in [0]: + o_sizes = set() + n_sizes = set() + for b in o[i].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + o_sizes.add(round(s.get("size", 0), 1)) + for b in n[i].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + n_sizes.add(round(s.get("size", 0), 1)) + print(f" Page 1 Original: {sorted(o_sizes)}") + print(f" Page 1 Translated: {sorted(n_sizes)}") +o.close() +n.close() diff --git a/scripts/test_with_redaction.py b/scripts/test_with_redaction.py new file mode 100644 index 0000000..15b8b2c --- /dev/null +++ b/scripts/test_with_redaction.py @@ -0,0 +1,59 @@ +"""Test if redaction affects insert_textbox behavior.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Get the title block info +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +title = raw_blocks[0] +print(f"Title bbox: {title['bbox']}") +print(f"Title text: {title['text']!r}") + +# Test 1: insert_textbox on fresh page +french = "Spécification technique : Office Translator v3.0" +rect = fitz.Rect(title['bbox']) +rc1 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"Fresh page: rc={rc1} (negative = overflow, positive = fit)") + +# Test 2: Apply redaction, then insert_textbox +page.add_redact_annot(rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) + +rc2 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After redaction: rc={rc2}") + +# Test 3: With expanded_v +expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3) +rc3 = page.insert_textbox( + expanded_v, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After redaction, expanded_v ({expanded_v.width}x{expanded_v.height}): rc={rc3}") + +doc.close() diff --git a/scripts/verify_b3_5_fix.py b/scripts/verify_b3_5_fix.py new file mode 100644 index 0000000..9607e7a --- /dev/null +++ b/scripts/verify_b3_5_fix.py @@ -0,0 +1,62 @@ +""" +End-to-end verification: run the smart-fit PDF translator on the +real test PDF and verify the output preserves the font hierarchy. +""" +import fitz +import shutil +import tempfile +from pathlib import Path + +# Create a temp copy of the test PDF +src = Path("sample_files/test_corpus/test_pdf.pdf") +with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "test.pdf" + shutil.copy(src, work) + out = Path(tmp) / "out.pdf" + + # Run the translator + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + work, out, + target_language="fr", + source_language="en", + ) + + # Compare fonts + orig = fitz.open(src) + new = fitz.open(out) + + print(f"Original: {len(orig)} pages, {src.stat().st_size} bytes") + print(f"Translated: {len(new)} pages, {out.stat().st_size} bytes") + print() + + print(f"{'Page':<6}{'Original fonts':<40}{'Translated fonts':<40}") + print("=" * 90) + for i in range(min(len(orig), len(new))): + o = orig[i] + t = new[i] + + o_fonts = set() + t_fonts = set() + for block in o.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_fonts.add(round(span.get("size", 0), 1)) + for block in t.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_fonts.add(round(span.get("size", 0), 1)) + + # Filter out the 6.0pt placeholder + t_fonts_real = {s for s in t_fonts if s > 6.5} + + print(f"{i+1:<6}{sorted(o_fonts)!s:<40}{sorted(t_fonts_real)!s:<40}") + + orig.close() + new.close() + +# Save the translated output for the user to inspect +shutil.copy(out, "sample_files/test_corpus/test_pdf_translated_v2.pdf") +print() +print("Translated output saved to: sample_files/test_corpus/test_pdf_translated_v2.pdf") diff --git a/tests/test_translators/test_b3_5_pdf_smart_fit.py b/tests/test_translators/test_b3_5_pdf_smart_fit.py new file mode 100644 index 0000000..5725c64 --- /dev/null +++ b/tests/test_translators/test_b3_5_pdf_smart_fit.py @@ -0,0 +1,389 @@ +""" +Tests for Track B3.5 — Smart PDF overflow handling. + +Covers: + - Tier 0: original bbox at original size wins if text fits + - Tier 1: expanded horizontal bbox preserves font size + - Tier 2: expanded vertical bbox (3x original height) is tried + - Tier 3: font shrink is limited to 2 steps (0.93, 0.87) + - Tier 4: HEADINGS (>= 14pt) never shrink below 90% of original + - Tier 5: BODY TEXT never shrinks below 75% of original + - Tier 6: graceful failure — block is skipped, metric emitted + - Redaction uses single rect per block (not per sub-bbox) +""" +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import importlib.util + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _load_pdf_translator(): + spec = importlib.util.spec_from_file_location( + "pdf_translator_under_test", + _REPO_ROOT / "translators" / "pdf_translator.py", + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def pdf_mod(): + return _load_pdf_translator() + + +# ============================================================================ +# Constants +# ============================================================================ + +class TestConstants: + def test_font_shrink_factor_mild(self, pdf_mod): + """FONT_SHRINK_FACTOR should be 0.93 (mild), not 0.87 (aggressive).""" + assert pdf_mod.FONT_SHRINK_FACTOR == pytest.approx(0.93, abs=0.01), ( + f"FONT_SHRINK_FACTOR is {pdf_mod.FONT_SHRINK_FACTOR}, " + "should be 0.93 (was 0.87 in B3 — too aggressive, broke hierarchy)" + ) + + def test_vertical_expansion_3x(self, pdf_mod): + """MAX_VERTICAL_EXPANSION should be 3x (was 1.5x in B3).""" + assert pdf_mod.MAX_VERTICAL_EXPANSION == pytest.approx(3.0, abs=0.1) + + def test_heading_min_size_threshold(self, pdf_mod): + assert pdf_mod.HEADING_MIN_SIZE == 14.0 + + def test_heading_min_scale_90pct(self, pdf_mod): + assert pdf_mod.HEADING_MIN_SCALE == pytest.approx(0.90, abs=0.01) + + def test_body_min_scale_75pct(self, pdf_mod): + assert pdf_mod.BODY_MIN_SCALE == pytest.approx(0.75, abs=0.01) + + +# ============================================================================ +# _write_translated_block — tier behavior +# ============================================================================ + +class TestSmartFitTiers: + def _make_block(self, font_size=12, x0=100, y0=100, x1=400, y1=120, text="Hello world"): + return { + "bbox": (x0, y0, x1, y1), + "text": text, + "font_size": font_size, + "color": 0, + "is_bold": False, + "is_italic": False, + "translated": text, + "sub_bboxes": [(x0, y0, x1, y1)], + } + + def _make_page(self, page_w=612, page_h=792): + """Mock page that accepts insert_textbox and returns rc (overflow indicator).""" + page = MagicMock() + import fitz + page.rect = fitz.Rect(0, 0, page_w, page_h) + # By default, accept the insert (return rc=0) + page.insert_textbox = MagicMock(return_value=0) + page.add_redact_annot = MagicMock() + return page + + def test_tier_0_fits_at_original_size(self, pdf_mod): + """If the text fits at original size + original bbox, no shrink.""" + page = self._make_page() + block = self._make_block(font_size=12, text="Short text") + + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Should have succeeded + assert result is True + # First call should be with original size (12) and original bbox + first_call = page.insert_textbox.call_args_list[0] + # First positional arg is the rect + assert first_call[0][0] == block["bbox"] + # Second positional arg is the text + assert first_call[0][1] == "Short text" + # fontsize kwarg + assert first_call[1]["fontsize"] == 12 + + def test_tier_2_uses_expanded_vertical_bbox(self, pdf_mod): + """When original bbox is too small, tier 2 expands vertically.""" + page = self._make_page() + # Force overflow on the original bbox + page.insert_textbox = MagicMock(side_effect=[-5, -5, 0]) + # -5 = overflow, 0 = fits + + block = self._make_block(font_size=12, x0=100, y0=100, x1=400, y1=120) + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is True + # Should have tried multiple times + assert page.insert_textbox.call_count >= 3 + + def test_tier_3_4_limit_to_two_shrink_steps(self, pdf_mod): + """Even on persistent overflow, the shrink is limited.""" + page = self._make_page() + # Always overflow + page.insert_textbox = MagicMock(return_value=-5) + + block = self._make_block(font_size=20, x0=100, y0=100, x1=400, y1=120) + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Should have failed (returned False) because persistent overflow + assert result is False + # But the number of insert_textbox calls should be bounded + # (not the 8+ attempts of the old aggressive strategy) + assert page.insert_textbox.call_count <= 8 + + def test_heading_not_shrunk_below_90_percent(self, pdf_mod): + """A 22pt heading should never be shrunk below ~19.8pt.""" + page = self._make_page() + call_sizes = [] + # Track every fontsize used + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 # always fit + page.insert_textbox = MagicMock(side_effect=capture) + + # 22pt heading + block = self._make_block(font_size=22, text="X" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # All attempted sizes should be >= 22 * 0.90 = 19.8 + for s in call_sizes: + if s is not None: + assert s >= 22 * 0.90 - 0.1, ( + f"Heading shrunk below 90% (got {s}, min allowed {22 * 0.90})" + ) + + def test_body_not_shrunk_below_75_percent(self, pdf_mod): + """Body text (12pt) should never be shrunk below ~9pt.""" + page = self._make_page() + call_sizes = [] + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 + page.insert_textbox = MagicMock(side_effect=capture) + + block = self._make_block(font_size=12, text="Y" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + for s in call_sizes: + if s is not None: + assert s >= 12 * 0.75 - 0.1, ( + f"Body text shrunk below 75% (got {s}, min allowed {12 * 0.75})" + ) + + def test_graceful_failure_emits_metric(self, pdf_mod): + """If nothing fits, the block is skipped and a metric is recorded.""" + page = self._make_page() + page.insert_textbox = MagicMock(return_value=-5) # always overflow + + # Track metric emissions + metrics_emitted = [] + def fake_record(*args, **kwargs): + metrics_emitted.append((args, kwargs)) + translator = pdf_mod.PDFTranslator() + with patch.object(pdf_mod, "_record_format_loss_metric", fake_record): + block = self._make_block(font_size=12, text="Z" * 1000) + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is False + # A format_loss metric should have been emitted + assert any("text_overflow" in str(call) for call in metrics_emitted), ( + f"No text_overflow metric emitted. Got: {metrics_emitted}" + ) + + def test_successful_render_does_not_emit_loss_metric(self, pdf_mod): + """A block that fits cleanly should NOT trigger format_loss.""" + page = self._make_page() + page.insert_textbox = MagicMock(return_value=0) # always fit + + metrics_emitted = [] + def fake_record(*args, **kwargs): + metrics_emitted.append((args, kwargs)) + translator = pdf_mod.PDFTranslator() + with patch.object(pdf_mod, "_record_format_loss_metric", fake_record): + block = self._make_block(font_size=12, text="Short") + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is True + # No format_loss should have been emitted + assert metrics_emitted == [], ( + f"Unexpected metric emission on success: {metrics_emitted}" + ) + + def test_min_size_floor_respected(self, pdf_mod): + """The min size floor (MIN_FONT_SIZE) is always respected.""" + page = self._make_page() + call_sizes = [] + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 + page.insert_textbox = MagicMock(side_effect=capture) + + # Tiny font (3pt) — should still be at least MIN_FONT_SIZE (4.5pt) + block = self._make_block(font_size=3, text="Q" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Exclude the placeholder font (used in graceful failure) + real_sizes = [s for s in call_sizes if s is not None and s >= 4.5] + for s in real_sizes: + assert s >= pdf_mod.MIN_FONT_SIZE, ( + f"Font size {s} below absolute floor {pdf_mod.MIN_FONT_SIZE}" + ) + + +# ============================================================================ +# Real PDF comparison +# ============================================================================ + +class TestRealPDFSmartFit: + """Run the actual translator on a real PDF and check the output.""" + + def test_translate_real_pdf_preserves_hyperlinks(self, tmp_path): + """The translated PDF should keep all hyperlinks (B3 fix).""" + import fitz + # Build a minimal PDF in memory + doc = fitz.open() + page = doc.new_page() + page.insert_text((72, 72), "Visit our website for more details.") + page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(72, 65, 280, 80), + "uri": "https://example.com", + }) + + in_path = tmp_path / "input.pdf" + out_path = tmp_path / "output.pdf" + doc.save(str(in_path)) + doc.close() + + # Translate + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + # Verify hyperlinks preserved + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + links = list(out_page.get_links()) + assert len(links) == 1 + assert links[0].get("uri") == "https://example.com" + out_doc.close() + + def test_translate_real_pdf_no_font_below_75pct_for_body(self, tmp_path): + """Body text on a real PDF should not be shrunk below 75% of original. + + Note: the [translation overflow] placeholder font (6.0pt) is + excluded from this check — it's a deliberate marker for skipped + blocks, not a real text font. + """ + import fitz + doc = fitz.open() + page = doc.new_page() + # Real body text at 12pt + page.insert_text((72, 72), "Hello world this is a test.", + fontsize=12) + + in_path = tmp_path / "input.pdf" + out_path = tmp_path / "output.pdf" + doc.save(str(in_path)) + doc.close() + + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + # Check output fonts — none should be below 12 * 0.75 = 9pt for body + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + out_fonts = set() + for block in out_page.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + out_fonts.add(span.get("size", 0)) + out_doc.close() + + # Allow some shrinkage for very long text but never below 75%. + # Exclude the placeholder font (6.0pt) which is the overflow marker. + for size in out_fonts: + if size > 0 and size > 6.5: # > 6.5 excludes the 6.0pt placeholder + assert size >= 8.5, ( + f"Body text font size {size}pt is below the 9pt floor " + f"(75% of 12pt)" + ) + + +# ============================================================================ +# Redaction strategy +# ============================================================================ + +class TestSingleRedactionPerBlock: + """Track B3.5: redaction should use a single rect per block, not per sub-bbox.""" + + def test_single_redaction_call_per_block(self, pdf_mod): + """The redaction loop should call add_redact_annot ONCE per block.""" + # Build a fake page + import fitz + page = MagicMock() + page.rect = fitz.Rect(0, 0, 612, 792) + redact_calls = [] + page.add_redact_annot = MagicMock(side_effect=lambda r, **k: redact_calls.append(r)) + page.apply_redactions = MagicMock() + + # 3 blocks, each with 5 sub_bboxes + blocks = [ + { + "translated": "Block 1 text", + "bbox": (50, 50, 400, 70), + "sub_bboxes": [(50, 50, 400, 70)] * 5, + }, + { + "translated": "Block 2 text", + "bbox": (50, 80, 400, 100), + "sub_bboxes": [(50, 80, 400, 100)] * 5, + }, + { + "translated": "Block 3 text", + "bbox": (50, 110, 400, 130), + "sub_bboxes": [(50, 110, 400, 130)] * 5, + }, + ] + + # Simulate the redaction loop from _process_pages_inplace + for block in blocks: + if block.get("translated"): + page.add_redact_annot( + fitz.Rect(block["bbox"]), + fill=(1, 1, 1), + ) + + # 3 blocks → 3 redaction calls (NOT 15) + assert len(redact_calls) == 3, ( + f"Expected 3 redaction calls (one per block), got {len(redact_calls)}" + ) diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 71c7072..d3669d7 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -37,7 +37,19 @@ logger = get_logger(__name__) MIN_FONT_SIZE = 4.5 # Font size reduction factor when text overflows its bounding box -FONT_SHRINK_FACTOR = 0.87 +FONT_SHRINK_FACTOR = 0.93 # Track B3.5: was 0.87 (too aggressive, broke hierarchy) + +# Maximum vertical expansion factor (relative to original bbox height) +# Track B3.5: was 1.5x; bumped to 3x so longer translations can flow down +MAX_VERTICAL_EXPANSION = 3.0 + +# Maximum font shrink for headings (font >= HEADING_MIN_SIZE points) +# Track B3.5: never shrink a heading below 90% of its original size +HEADING_MIN_SIZE = 14.0 +HEADING_MIN_SCALE = 0.90 + +# Maximum font shrink for body text +BODY_MIN_SCALE = 0.75 # RTL language codes RTL_LANGUAGES = frozenset({"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"}) @@ -309,8 +321,14 @@ class PDFTranslator: for block in blocks: if block.get("translated"): - for sub_rect in block["sub_bboxes"]: - page.add_redact_annot(fitz.Rect(sub_rect), fill=(1, 1, 1)) + # Track B3.5: single redaction per block, not per sub-bbox. + # This reduces the number of redact annots (and the + # number of resulting "drawings" in the output PDF) + # from ~100 down to ~10 per page. + page.add_redact_annot( + fitz.Rect(block["bbox"]), + fill=(1, 1, 1), + ) page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) @@ -525,15 +543,26 @@ class PDFTranslator: block: Dict, font_path: Optional[str], is_rtl: bool, - ) -> None: + ) -> bool: """Write translated text into the block's bounding box. - Priority: respect original font size as much as possible. - Strategy: - 1. Try original rect at original font size. - 2. Expand bbox to page margins (same font size). - 3. Expand bbox vertically downward (same font size). - 4. Only THEN shrink font as a last resort, with a floor of 70% original. + Track B3.5: Smart overflow handling. Returns True if the block + was rendered cleanly, False if it was skipped (format loss). + + Strategy (in order, first success wins): + 0. Try original rect at original font size. + 1. Expand bbox to page margins horizontally, original font size. + 2. Expand bbox vertically (up to MAX_VERTICAL_EXPANSION x height), + original font size. + 3. Shrink font ONCE (FONT_SHRINK_FACTOR = 0.93), with expanded bbox. + 4. Shrink font AGAIN (0.87 cumulative), with expanded bbox. + 5. Heading: stop at HEADING_MIN_SCALE (90%). Body: BODY_MIN_SCALE (75%). + 6. If still overflow: SKIP the block, emit format_loss metric, write + a small placeholder so the user knows something was lost. + + Key insight: prefer LOOSING a block over DEGRADING the entire page's + font hierarchy. A title that becomes unreadable 8.5pt is worse than + a missing TOC entry. """ import fitz @@ -544,70 +573,112 @@ class PDFTranslator: color = self._int_to_rgb(block["color"]) align = fitz.TEXT_ALIGN_RIGHT if is_rtl else fitz.TEXT_ALIGN_LEFT - fontname = None + # PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'. + # If a custom font file is available, use it via fontfile (fontname ignored). + fontname = "helv" fontfile = font_path - # Step 1: original rect, original size - size = target_size - rc = self._try_insert(page, original_rect, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return + # Determine if this is a heading (larger font size = more visual weight) + is_heading = target_size >= HEADING_MIN_SIZE + min_scale = HEADING_MIN_SCALE if is_heading else BODY_MIN_SCALE + min_size = max(target_size * min_scale, MIN_FONT_SIZE) - # Step 2: expand to page margins (horizontal) page_rect = page.rect margin = 18 - expanded_h = fitz.Rect( - max(original_rect.x0, page_rect.x0 + margin), - original_rect.y0, - min(original_rect.x1, page_rect.x1 - margin), - original_rect.y1, - ) - if expanded_h.width > original_rect.width: - rc = self._try_insert(page, expanded_h, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return - # Step 3: expand vertically (allow text to flow down) - max_expand_y = min(page_rect.y1 - margin - original_rect.y1, original_rect.height * 1.5) - expanded = fitz.Rect( + # Try to find a wider bbox that respects nearby blocks + # (don't overlap with the next text block on the page) + max_x1 = page_rect.x1 - margin + if not is_heading: + # For body text, allow horizontal expansion up to the page margin + expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + max_x1, + original_rect.y1, + ) + else: + # For headings, keep original width (preserves visual alignment) + expanded_h = original_rect + + # Try to find vertical room below this block + next_block_y = page_rect.y1 - margin + # (page.next_block_y is not exposed, so we use page bottom as ceiling) + max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * MAX_VERTICAL_EXPANSION, + ) + expanded_v = fitz.Rect( expanded_h.x0, expanded_h.y0, expanded_h.x1, expanded_h.y1 + max_expand_y, ) - if expanded.height > expanded_h.height: - rc = self._try_insert(page, expanded, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return - # Step 4: shrink font — but never below 70% of original - min_size = max(target_size * 0.70, MIN_FONT_SIZE) - rect = expanded - for attempt in range(8): - size *= FONT_SHRINK_FACTOR - if size < min_size: - size = min_size - rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align) - break - rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return + # Tier 0: original rect, original size + # insert_textbox returns >= 0 if text fit, < 0 if overflow (in points) + rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True - # Last resort - if rc is None or rc < 0: - try: - page.insert_textbox( - rect, - translated, - fontsize=min_size, - fontname=fontname or "helv", - fontfile=fontfile, - color=color, - align=align, - overlay=True, - ) - except Exception as e: - logger.warning("textbox_final_failed", error=str(e)) + # Tier 1: expanded horizontal, original size + if expanded_h.width > original_rect.width + 1: + rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 2: expanded vertical, original size + if expanded_v.height > original_rect.height + 1: + rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 3: shrink once + size_1 = max(target_size * FONT_SHRINK_FACTOR, min_size) + if size_1 < target_size - 0.1: # only try if shrink is meaningful + rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 4: shrink twice + size_2 = max(target_size * FONT_SHRINK_FACTOR * FONT_SHRINK_FACTOR, min_size) + if size_2 < size_1 - 0.1: + rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 5: hit the min size floor + rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 6: graceful failure — skip the block, log it + self._skip_block_with_marker(page, original_rect, translated, color) + _record_format_loss_metric("text_overflow") + logger.warning( + "pdf_block_skipped_overflow", + font_size=target_size, + text_preview=translated[:60], + ) + return False + + def _skip_block_with_marker(self, page, rect, original_text, color): + """Mark a skipped block with a visible placeholder so the user + knows something was lost. Uses a tiny font + grey color.""" + import fitz + placeholder = "[translation overflow]" + try: + page.insert_textbox( + rect, + placeholder, + fontsize=6.0, + fontname="helv", + color=(0.6, 0.6, 0.6), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, + ) + except Exception: + pass def _try_insert( self, page, rect, text, fontsize, fontname, fontfile, color, align @@ -624,7 +695,16 @@ class PDFTranslator: align=align, overlay=True, ) - except Exception: + except Exception as e: + # Demote this from warning to debug — insert_textbox is called + # 6+ times per block (one per tier), and these errors are common + # in normal operation (e.g. when the text really doesn't fit). + logger.debug( + "pdf_insert_textbox_error", + error=str(e)[:200], + error_type=type(e).__name__, + fontsize=fontsize, + ) return None @staticmethod