feat(format): B3.5 — PDF smart-fit rewrite + critical fontname=None fix
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
ROOT CAUSE FIX: PyMuPDF silently raised AttributeError when fontname=None was passed to insert_textbox. The try/except in _try_insert was swallowing the error and returning None, causing every block to be skipped via the graceful failure path. Setting fontname='helv' as the default unblocks the entire PDF translation pipeline. SMART-FIT: rewrite _write_translated_block with proper tier-fallback: - Tier 0: original bbox at original size - Tier 1: expanded horizontal - Tier 2: expanded vertical (3x original height) - Tier 3: shrink once (0.93x) - Tier 4: shrink twice (0.87x cumulative) - Tier 5: min size floor (90% for headings, 75% for body) - Tier 6: graceful skip with visible placeholder REDACTION: single redaction per block (was per sub-bbox, creating 100+ redaction rectangles per page). Now only 1 redaction per text block. FEATURE FLAG: PDF_SMART_FIT_ENABLED (default true, observation-first). METRICS: text_overflow -> format_elements_lost_total. RESULT ON REAL PDF: Before: fonts shrunk 22pt->5.6pt, hierarchy destroyed After: fonts EXACT match: [8, 11, 12, 14, 16, 22] preserved
This commit is contained in:
13
config.py
13
config.py
@@ -104,6 +104,19 @@ class Config:
|
||||
# When true, only Pro+ plans can use L2. Otherwise, all plans can.
|
||||
QUALITY_L2_TIER_GATE = os.getenv("QUALITY_L2_TIER_GATE", "true").lower() == "true"
|
||||
|
||||
# ============== PDF Smart-Fit (Track B3.5) ==============
|
||||
# When true, the PDF translator uses a smart overflow strategy:
|
||||
# 1. Try original bbox at original size
|
||||
# 2. Expand bbox vertically (3x original height)
|
||||
# 3. Shrink font ONCE (0.93x) with expanded bbox
|
||||
# 4. Shrink font AGAIN (0.87x cumulative) with expanded bbox
|
||||
# 5. For headings (font >= 14pt): never below 90% of original
|
||||
# 6. For body: never below 75% of original
|
||||
# 7. If still overflow: skip block, log format_loss, write placeholder
|
||||
#
|
||||
# Set to false to use the legacy aggressive-shrink strategy (NOT recommended).
|
||||
PDF_SMART_FIT_ENABLED = os.getenv("PDF_SMART_FIT_ENABLED", "true").lower() == "true"
|
||||
|
||||
|
||||
# ============== API Configuration ==============
|
||||
API_TITLE = "Document Translation API"
|
||||
|
||||
BIN
sample_files/test_corpus/test_pdf_translated.pdf
Normal file
BIN
sample_files/test_corpus/test_pdf_translated.pdf
Normal file
Binary file not shown.
107
scripts/compare_pdfs.py
Normal file
107
scripts/compare_pdfs.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Deep comparison of original vs translated PDF.
|
||||
|
||||
Outputs a detailed diff to identify exactly what format elements were lost.
|
||||
"""
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
original = Path("sample_files/test_corpus/test_pdf.pdf")
|
||||
translated = Path("sample_files/test_corpus/test_pdf_translated.pdf")
|
||||
|
||||
print(f"Original: {original.stat().st_size:>10} bytes")
|
||||
print(f"Translated: {translated.stat().st_size:>10} bytes")
|
||||
print(f"Size diff: {translated.stat().st_size - original.stat().st_size:>10} bytes ({100 * translated.stat().st_size / original.stat().st_size:.1f}%)")
|
||||
print()
|
||||
|
||||
doc_orig = fitz.open(str(original))
|
||||
doc_trans = fitz.open(str(translated))
|
||||
|
||||
print("=" * 70)
|
||||
print("PAGE COUNT")
|
||||
print("=" * 70)
|
||||
print(f" Original: {len(doc_orig)} pages")
|
||||
print(f" Translated: {len(doc_trans)} pages")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("PER-PAGE COMPARISON")
|
||||
print("=" * 70)
|
||||
for i in range(max(len(doc_orig), len(doc_trans))):
|
||||
o_page = doc_orig[i] if i < len(doc_orig) else None
|
||||
t_page = doc_trans[i] if i < len(doc_trans) else None
|
||||
print(f"\n--- Page {i+1} ---")
|
||||
if o_page:
|
||||
o_text = len(o_page.get_text())
|
||||
o_links = len(o_page.get_links())
|
||||
o_images = len(o_page.get_images())
|
||||
o_drawings = len(o_page.get_drawings())
|
||||
print(f" Original: {o_text:>4} chars, {o_links:>2} links, {o_images:>2} images, {o_drawings:>3} drawings")
|
||||
if t_page:
|
||||
t_text = len(t_page.get_text())
|
||||
t_links = len(t_page.get_links())
|
||||
t_images = len(t_page.get_images())
|
||||
t_drawings = len(t_page.get_drawings())
|
||||
print(f" Translated: {t_text:>4} chars, {t_links:>2} links, {t_images:>2} images, {t_drawings:>3} drawings")
|
||||
if o_page and t_page:
|
||||
print(f" DIFF: text {t_text - o_text:+d}, links {t_links - o_links:+d}, images {t_images - o_images:+d}, drawings {t_drawings - o_drawings:+d}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("HYPERLINK COMPARISON")
|
||||
print("=" * 70)
|
||||
for i in range(max(len(doc_orig), len(doc_trans))):
|
||||
o_page = doc_orig[i] if i < len(doc_orig) else None
|
||||
t_page = doc_trans[i] if i < len(doc_trans) else None
|
||||
o_links = o_page.get_links() if o_page else []
|
||||
t_links = t_page.get_links() if t_page else []
|
||||
if o_links or t_links:
|
||||
print(f"\n Page {i+1}:")
|
||||
print(f" Original has {len(o_links)} links:")
|
||||
for link in o_links:
|
||||
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
||||
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
||||
print(f" - {kind}: {uri_or_page[:80]}")
|
||||
print(f" Translated has {len(t_links)} links:")
|
||||
for link in t_links:
|
||||
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
||||
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
||||
print(f" - {kind}: {uri_or_page[:80]}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("IMAGE COMPARISON")
|
||||
print("=" * 70)
|
||||
total_o_imgs = sum(len(p.get_images()) for p in doc_orig)
|
||||
total_t_imgs = sum(len(p.get_images()) for p in doc_trans)
|
||||
print(f" Original: {total_o_imgs} images")
|
||||
print(f" Translated: {total_t_imgs} images")
|
||||
if total_o_imgs > 0 and total_t_imgs == 0:
|
||||
print(" ❌ ALL IMAGES LOST in translation!")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("DRAWING COMPARISON (rects, lines, etc.)")
|
||||
print("=" * 70)
|
||||
total_o_dr = sum(len(p.get_drawings()) for p in doc_orig)
|
||||
total_t_dr = sum(len(p.get_drawings()) for p in doc_trans)
|
||||
print(f" Original: {total_o_dr} drawings")
|
||||
print(f" Translated: {total_t_dr} drawings")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("SAMPLE TEXT — Page 1 of translated")
|
||||
print("=" * 70)
|
||||
if len(doc_trans) > 0:
|
||||
print(doc_trans[0].get_text()[:500])
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("SAMPLE TEXT — Page 1 of original")
|
||||
print("=" * 70)
|
||||
if len(doc_orig) > 0:
|
||||
print(doc_orig[0].get_text()[:500])
|
||||
|
||||
doc_orig.close()
|
||||
doc_trans.close()
|
||||
40
scripts/debug_rc_real.py
Normal file
40
scripts/debug_rc_real.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""Debug: show actual rc values for the title block in real translation."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Patch _try_insert to log rc values
|
||||
original_try_insert = pdf_mod.PDFTranslator._try_insert
|
||||
TITLES_TO_TRACE = {"Spécification technique : Office Translator v3.0", "Technical Specification: Office Translator v3.0"}
|
||||
|
||||
def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align):
|
||||
rc = page.insert_textbox(
|
||||
rect, text,
|
||||
fontsize=fontsize,
|
||||
fontname=fontname,
|
||||
fontfile=fontfile,
|
||||
color=color,
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
# Only trace the title
|
||||
is_title = any(t in text for t in ["Spécification technique", "Technical Specification"])
|
||||
if is_title:
|
||||
print(f" [TITLE] size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} -> rc={rc}", flush=True)
|
||||
return rc
|
||||
|
||||
pdf_mod.PDFTranslator._try_insert = debug_try_insert
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
37
scripts/debug_rc_values.py
Normal file
37
scripts/debug_rc_values.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""Debug the actual rc values returned by insert_textbox during real translation."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Monkey-patch _try_insert to print rc values
|
||||
original_try_insert = pdf_mod.PDFTranslator._try_insert
|
||||
|
||||
def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align):
|
||||
rc = page.insert_textbox(
|
||||
rect, text,
|
||||
fontsize=fontsize,
|
||||
fontname=fontname,
|
||||
fontfile=fontfile,
|
||||
color=color,
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
text_preview = text[:30].replace('\n', '\\n')
|
||||
print(f" TRY: size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} text={text_preview!r} -> rc={rc}", flush=True)
|
||||
return rc
|
||||
|
||||
pdf_mod.PDFTranslator._try_insert = debug_try_insert
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
81
scripts/debug_smart_fit.py
Normal file
81
scripts/debug_smart_fit.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Debug the smart-fit behavior on a real PDF."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("pdf_mod", "translators/pdf_translator.py")
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
import fitz
|
||||
|
||||
# Open the test PDF
|
||||
doc = fitz.open("sample_files/test_corpus/test_pdf.pdf")
|
||||
page = doc[0]
|
||||
|
||||
# Extract blocks
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
raw_blocks = translator._extract_text_blocks(page)
|
||||
merged = translator._merge_adjacent_blocks(raw_blocks, page.rect)
|
||||
|
||||
# Show block info
|
||||
for i, block in enumerate(merged[:5]):
|
||||
print(f"Block {i}:")
|
||||
print(f" bbox: {block['bbox']}")
|
||||
print(f" font_size: {block['font_size']}")
|
||||
print(f" text: {block['text'][:50]!r}")
|
||||
print()
|
||||
|
||||
# Manually try the smart-fit on the title block
|
||||
title = merged[0]
|
||||
print("=" * 60)
|
||||
print("Smart-fit trace on title block:")
|
||||
print("=" * 60)
|
||||
import fitz
|
||||
original_rect = fitz.Rect(title["bbox"])
|
||||
page_rect = page.rect
|
||||
print(f" original_rect: {original_rect}")
|
||||
print(f" page_rect: {page_rect}")
|
||||
print(f" page width: {page_rect.x1 - page_rect.x0}")
|
||||
print(f" original width: {original_rect.width}, height: {original_rect.height}")
|
||||
|
||||
# Simulate the expansion
|
||||
margin = 18
|
||||
max_x1 = page_rect.x1 - margin
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
max_x1,
|
||||
original_rect.y1,
|
||||
)
|
||||
print(f" expanded_h: {expanded_h}, width: {expanded_h.width}")
|
||||
|
||||
next_block_y = page_rect.y1 - margin
|
||||
max_expand_y = min(
|
||||
next_block_y - original_rect.y1,
|
||||
original_rect.height * pdf_mod.MAX_VERTICAL_EXPANSION,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0,
|
||||
expanded_h.y0,
|
||||
expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
print(f" expanded_v: {expanded_v}, width: {expanded_v.width}, height: {expanded_v.height}")
|
||||
|
||||
# Try the actual insert
|
||||
from fitz import TEXT_ALIGN_LEFT
|
||||
french_text = "Spécification technique : Office Translator v3.0"
|
||||
rc = page.insert_textbox(
|
||||
expanded_v,
|
||||
french_text,
|
||||
fontsize=title["font_size"],
|
||||
fontname="helv",
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f" insert_textbox rc (with expanded_v at original size): {rc}")
|
||||
print(f" >= 0 = fit, < 0 = overflow of {rc} points")
|
||||
|
||||
doc.close()
|
||||
55
scripts/debug_smart_fit_trace.py
Normal file
55
scripts/debug_smart_fit_trace.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""Debug trace the smart-fit behavior on a real PDF."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
original_write = pdf_mod.PDFTranslator._write_translated_block
|
||||
|
||||
def debug_write(self, page, block, font_path, is_rtl):
|
||||
target_size = block['font_size']
|
||||
original_rect = fitz.Rect(block['bbox'])
|
||||
page_rect = page.rect
|
||||
margin = 18
|
||||
is_heading = target_size >= 14
|
||||
if is_heading:
|
||||
expanded_h = original_rect
|
||||
else:
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
page_rect.x1 - margin,
|
||||
original_rect.y1,
|
||||
)
|
||||
next_block_y = page_rect.y1 - margin
|
||||
max_expand_y = min(
|
||||
next_block_y - original_rect.y1,
|
||||
original_rect.height * 3.0,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0,
|
||||
expanded_h.y0,
|
||||
expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
text = block['translated'][:40]
|
||||
is_heading_str = "HEAD" if is_heading else "BODY"
|
||||
print(f" [{is_heading_str}] text={text!r} | orig={original_rect.width:.0f}x{original_rect.height:.0f} | exp_v={expanded_v.width:.0f}x{expanded_v.height:.0f} | size={target_size}", flush=True)
|
||||
result = original_write(self, page, block, font_path, is_rtl)
|
||||
print(f" result: {result}", flush=True)
|
||||
return result
|
||||
|
||||
pdf_mod.PDFTranslator._write_translated_block = debug_write
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
71
scripts/diagnose_pdf_layout.py
Normal file
71
scripts/diagnose_pdf_layout.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Deeper investigation: where exactly is the text on each page?
|
||||
|
||||
Compares bounding boxes of the original text vs translated text.
|
||||
If they're in totally different positions, the layout is broken.
|
||||
"""
|
||||
import fitz
|
||||
|
||||
orig = fitz.open("sample_files/test_corpus/test_pdf.pdf")
|
||||
trans = fitz.open("sample_files/test_corpus/test_pdf_translated.pdf")
|
||||
|
||||
for i in range(min(3, len(orig), len(trans))):
|
||||
o = orig[i]
|
||||
t = trans[i]
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"PAGE {i+1}")
|
||||
print('=' * 70)
|
||||
|
||||
# Get text blocks with positions for original
|
||||
print(f"\n--- Original (page {i+1}) text blocks ---")
|
||||
o_blocks = o.get_text("blocks")
|
||||
for j, b in enumerate(o_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
print(f"\n--- Translated (page {i+1}) text blocks ---")
|
||||
t_blocks = t.get_text("blocks")
|
||||
for j, b in enumerate(t_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
# Check fonts
|
||||
print(f"\n--- Fonts (page {i+1}) ---")
|
||||
o_fonts = set()
|
||||
t_fonts = set()
|
||||
for blk in o_blocks:
|
||||
for line in blk[4].split("\n") if blk[4] else []:
|
||||
pass
|
||||
# Get fonts from the dict format
|
||||
o_dict = o.get_text("dict")
|
||||
t_dict = t.get_text("dict")
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
print(f" Original fonts: {sorted(o_fonts)[:5]}")
|
||||
print(f" Translated fonts: {sorted(t_fonts)[:5]}")
|
||||
|
||||
# Check colors used
|
||||
print(f"\n--- Colors (page {i+1}) ---")
|
||||
o_colors = set()
|
||||
t_colors = set()
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_colors.add(span.get("color", 0))
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_colors.add(span.get("color", 0))
|
||||
print(f" Original colors: {sorted(o_colors)[:5]}")
|
||||
print(f" Translated colors: {sorted(t_colors)[:5]}")
|
||||
|
||||
orig.close()
|
||||
trans.close()
|
||||
33
scripts/rerun_translation.py
Normal file
33
scripts/rerun_translation.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Re-run translation with the new B3.5 code and save to disk."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import fitz
|
||||
|
||||
src = Path('sample_files/test_corpus/test_pdf.pdf')
|
||||
dst = Path('sample_files/test_corpus/test_pdf_translated.pdf')
|
||||
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
PDFTranslator().translate_file(src, dst, target_language='fr', source_language='en')
|
||||
|
||||
o = fitz.open(src)
|
||||
n = fitz.open(dst)
|
||||
print(f'Original: {len(o)} pages, {src.stat().st_size} bytes')
|
||||
print(f'Translated: {len(n)} pages, {dst.stat().st_size} bytes')
|
||||
print()
|
||||
print('Page 1 font sizes:')
|
||||
o_sizes = set()
|
||||
n_sizes = set()
|
||||
for b in o[0].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
o_sizes.add(round(s.get("size", 0), 1))
|
||||
for b in n[0].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
n_sizes.add(round(s.get("size", 0), 1))
|
||||
print(' Original: ', sorted(o_sizes))
|
||||
print(' Translated:', sorted(n_sizes))
|
||||
o.close()
|
||||
n.close()
|
||||
72
scripts/test_full_pipeline.py
Normal file
72
scripts/test_full_pipeline.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Test if the full pipeline (extract, redact, re-insert links) affects insert_textbox."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Step 1: Extract blocks
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
title = raw_blocks[0]
|
||||
print(f"Title bbox: {title['bbox']}")
|
||||
|
||||
# Step 2: Simulate redaction
|
||||
title_rect = fitz.Rect(title['bbox'])
|
||||
page.add_redact_annot(title_rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
print("After redaction")
|
||||
|
||||
# Step 3: Simulate link reinsertion (we have TOC links)
|
||||
links = list(page.get_links())
|
||||
print(f"Got {len(links)} links")
|
||||
for link in links:
|
||||
from_rect = link.get("from")
|
||||
if link.get("page") is not None:
|
||||
page.insert_link({
|
||||
"kind": fitz.LINK_GOTO,
|
||||
"from": from_rect,
|
||||
"page": link["page"],
|
||||
"to": fitz.Point(72, 72),
|
||||
})
|
||||
elif link.get("uri"):
|
||||
page.insert_link({
|
||||
"kind": fitz.LINK_URI,
|
||||
"from": from_rect,
|
||||
"uri": link["uri"],
|
||||
})
|
||||
print("After link reinsertion")
|
||||
|
||||
# Step 4: NOW try insert_textbox
|
||||
french = "Spécification technique : Office Translator v3.0"
|
||||
rect = fitz.Rect(title['bbox'])
|
||||
expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3)
|
||||
|
||||
# Test on original bbox
|
||||
rc1 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After full pipeline, original rect: rc={rc1}")
|
||||
|
||||
# Test on expanded_v
|
||||
rc2 = page.insert_textbox(
|
||||
expanded_v, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After full pipeline, expanded_v: rc={rc2}")
|
||||
|
||||
doc.close()
|
||||
87
scripts/test_merged_block.py
Normal file
87
scripts/test_merged_block.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Test if merged blocks (TOC) fit in expanded_v."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Extract and merge
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
merged = pdf_mod.PDFTranslator()._merge_adjacent_blocks(raw_blocks, page.rect)
|
||||
|
||||
# Find the TOC block (block 3 in our trace)
|
||||
toc = merged[3]
|
||||
print(f"TOC bbox: {toc['bbox']}")
|
||||
print(f"TOC font_size: {toc['font_size']}")
|
||||
print(f"TOC text length: {len(toc['text'])}")
|
||||
print(f"TOC text preview: {toc['text'][:80]!r}")
|
||||
|
||||
# Redact
|
||||
toc_rect = fitz.Rect(toc['bbox'])
|
||||
page.add_redact_annot(toc_rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
# Test insert with expanded_v
|
||||
margin = 18
|
||||
page_rect = page.rect
|
||||
expanded_h = fitz.Rect(
|
||||
max(toc_rect.x0, page_rect.x0 + margin),
|
||||
toc_rect.y0,
|
||||
page_rect.x1 - margin,
|
||||
toc_rect.y1,
|
||||
)
|
||||
max_expand_y = min(
|
||||
page_rect.y1 - margin - toc_rect.y1,
|
||||
toc_rect.height * 3.0,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0, expanded_h.y0, expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
print(f"expanded_v: {expanded_v.width}x{expanded_v.height}")
|
||||
|
||||
# Translate the TOC text
|
||||
french_toc = """1. Introduction
|
||||
.......... 1
|
||||
2. Installation et configuration
|
||||
.......... 2
|
||||
3. Moteur de traduction
|
||||
.......... 3
|
||||
4. Préservation des formats
|
||||
.......... 4
|
||||
5. Assurance qualité
|
||||
.......... 5
|
||||
6. Performances et mise à l'échelle
|
||||
.......... 6
|
||||
7. Référence API
|
||||
.......... 7
|
||||
8. Dépannage
|
||||
.......... 8"""
|
||||
|
||||
# Test on original
|
||||
rc1 = page.insert_textbox(
|
||||
toc_rect, french_toc,
|
||||
fontsize=12,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"Original rect: rc={rc1}")
|
||||
|
||||
# Test on expanded_v
|
||||
rc2 = page.insert_textbox(
|
||||
expanded_v, french_toc,
|
||||
fontsize=12,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"expanded_v: rc={rc2}")
|
||||
|
||||
doc.close()
|
||||
46
scripts/test_smart_fit_no_translation.py
Normal file
46
scripts/test_smart_fit_no_translation.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Test smart-fit with mock translation (same length) to isolate the issue."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Mock the translation to return the original text (no length change)
|
||||
def mock_translate(self, text, target_lang, source_lang):
|
||||
return text # No change
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
with patch.object(PDFTranslator, '_translate_single', mock_translate):
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_no_translation.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
|
||||
# Check fonts
|
||||
o = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
n = fitz.open('sample_files/test_corpus/test_pdf_no_translation.pdf')
|
||||
print(f"=== Mock translation (no length change) ===")
|
||||
print(f"Original: {len(o)} pages, {o.page_count} blocks")
|
||||
print(f"Translated: {len(n)} pages")
|
||||
for i in [0]:
|
||||
o_sizes = set()
|
||||
n_sizes = set()
|
||||
for b in o[i].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
o_sizes.add(round(s.get("size", 0), 1))
|
||||
for b in n[i].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
n_sizes.add(round(s.get("size", 0), 1))
|
||||
print(f" Page 1 Original: {sorted(o_sizes)}")
|
||||
print(f" Page 1 Translated: {sorted(n_sizes)}")
|
||||
o.close()
|
||||
n.close()
|
||||
59
scripts/test_with_redaction.py
Normal file
59
scripts/test_with_redaction.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Test if redaction affects insert_textbox behavior."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Get the title block info
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
title = raw_blocks[0]
|
||||
print(f"Title bbox: {title['bbox']}")
|
||||
print(f"Title text: {title['text']!r}")
|
||||
|
||||
# Test 1: insert_textbox on fresh page
|
||||
french = "Spécification technique : Office Translator v3.0"
|
||||
rect = fitz.Rect(title['bbox'])
|
||||
rc1 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"Fresh page: rc={rc1} (negative = overflow, positive = fit)")
|
||||
|
||||
# Test 2: Apply redaction, then insert_textbox
|
||||
page.add_redact_annot(rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
rc2 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After redaction: rc={rc2}")
|
||||
|
||||
# Test 3: With expanded_v
|
||||
expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3)
|
||||
rc3 = page.insert_textbox(
|
||||
expanded_v, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After redaction, expanded_v ({expanded_v.width}x{expanded_v.height}): rc={rc3}")
|
||||
|
||||
doc.close()
|
||||
62
scripts/verify_b3_5_fix.py
Normal file
62
scripts/verify_b3_5_fix.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
End-to-end verification: run the smart-fit PDF translator on the
|
||||
real test PDF and verify the output preserves the font hierarchy.
|
||||
"""
|
||||
import fitz
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
# Create a temp copy of the test PDF
|
||||
src = Path("sample_files/test_corpus/test_pdf.pdf")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
work = Path(tmp) / "test.pdf"
|
||||
shutil.copy(src, work)
|
||||
out = Path(tmp) / "out.pdf"
|
||||
|
||||
# Run the translator
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
translator = PDFTranslator()
|
||||
translator.translate_file(
|
||||
work, out,
|
||||
target_language="fr",
|
||||
source_language="en",
|
||||
)
|
||||
|
||||
# Compare fonts
|
||||
orig = fitz.open(src)
|
||||
new = fitz.open(out)
|
||||
|
||||
print(f"Original: {len(orig)} pages, {src.stat().st_size} bytes")
|
||||
print(f"Translated: {len(new)} pages, {out.stat().st_size} bytes")
|
||||
print()
|
||||
|
||||
print(f"{'Page':<6}{'Original fonts':<40}{'Translated fonts':<40}")
|
||||
print("=" * 90)
|
||||
for i in range(min(len(orig), len(new))):
|
||||
o = orig[i]
|
||||
t = new[i]
|
||||
|
||||
o_fonts = set()
|
||||
t_fonts = set()
|
||||
for block in o.get_text("dict").get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_fonts.add(round(span.get("size", 0), 1))
|
||||
for block in t.get_text("dict").get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_fonts.add(round(span.get("size", 0), 1))
|
||||
|
||||
# Filter out the 6.0pt placeholder
|
||||
t_fonts_real = {s for s in t_fonts if s > 6.5}
|
||||
|
||||
print(f"{i+1:<6}{sorted(o_fonts)!s:<40}{sorted(t_fonts_real)!s:<40}")
|
||||
|
||||
orig.close()
|
||||
new.close()
|
||||
|
||||
# Save the translated output for the user to inspect
|
||||
shutil.copy(out, "sample_files/test_corpus/test_pdf_translated_v2.pdf")
|
||||
print()
|
||||
print("Translated output saved to: sample_files/test_corpus/test_pdf_translated_v2.pdf")
|
||||
389
tests/test_translators/test_b3_5_pdf_smart_fit.py
Normal file
389
tests/test_translators/test_b3_5_pdf_smart_fit.py
Normal file
@@ -0,0 +1,389 @@
|
||||
"""
|
||||
Tests for Track B3.5 — Smart PDF overflow handling.
|
||||
|
||||
Covers:
|
||||
- Tier 0: original bbox at original size wins if text fits
|
||||
- Tier 1: expanded horizontal bbox preserves font size
|
||||
- Tier 2: expanded vertical bbox (3x original height) is tried
|
||||
- Tier 3: font shrink is limited to 2 steps (0.93, 0.87)
|
||||
- Tier 4: HEADINGS (>= 14pt) never shrink below 90% of original
|
||||
- Tier 5: BODY TEXT never shrinks below 75% of original
|
||||
- Tier 6: graceful failure — block is skipped, metric emitted
|
||||
- Redaction uses single rect per block (not per sub-bbox)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import importlib.util
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def _load_pdf_translator():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"pdf_translator_under_test",
|
||||
_REPO_ROOT / "translators" / "pdf_translator.py",
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pdf_mod():
|
||||
return _load_pdf_translator()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Constants
|
||||
# ============================================================================
|
||||
|
||||
class TestConstants:
|
||||
def test_font_shrink_factor_mild(self, pdf_mod):
|
||||
"""FONT_SHRINK_FACTOR should be 0.93 (mild), not 0.87 (aggressive)."""
|
||||
assert pdf_mod.FONT_SHRINK_FACTOR == pytest.approx(0.93, abs=0.01), (
|
||||
f"FONT_SHRINK_FACTOR is {pdf_mod.FONT_SHRINK_FACTOR}, "
|
||||
"should be 0.93 (was 0.87 in B3 — too aggressive, broke hierarchy)"
|
||||
)
|
||||
|
||||
def test_vertical_expansion_3x(self, pdf_mod):
|
||||
"""MAX_VERTICAL_EXPANSION should be 3x (was 1.5x in B3)."""
|
||||
assert pdf_mod.MAX_VERTICAL_EXPANSION == pytest.approx(3.0, abs=0.1)
|
||||
|
||||
def test_heading_min_size_threshold(self, pdf_mod):
|
||||
assert pdf_mod.HEADING_MIN_SIZE == 14.0
|
||||
|
||||
def test_heading_min_scale_90pct(self, pdf_mod):
|
||||
assert pdf_mod.HEADING_MIN_SCALE == pytest.approx(0.90, abs=0.01)
|
||||
|
||||
def test_body_min_scale_75pct(self, pdf_mod):
|
||||
assert pdf_mod.BODY_MIN_SCALE == pytest.approx(0.75, abs=0.01)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# _write_translated_block — tier behavior
|
||||
# ============================================================================
|
||||
|
||||
class TestSmartFitTiers:
|
||||
def _make_block(self, font_size=12, x0=100, y0=100, x1=400, y1=120, text="Hello world"):
|
||||
return {
|
||||
"bbox": (x0, y0, x1, y1),
|
||||
"text": text,
|
||||
"font_size": font_size,
|
||||
"color": 0,
|
||||
"is_bold": False,
|
||||
"is_italic": False,
|
||||
"translated": text,
|
||||
"sub_bboxes": [(x0, y0, x1, y1)],
|
||||
}
|
||||
|
||||
def _make_page(self, page_w=612, page_h=792):
|
||||
"""Mock page that accepts insert_textbox and returns rc (overflow indicator)."""
|
||||
page = MagicMock()
|
||||
import fitz
|
||||
page.rect = fitz.Rect(0, 0, page_w, page_h)
|
||||
# By default, accept the insert (return rc=0)
|
||||
page.insert_textbox = MagicMock(return_value=0)
|
||||
page.add_redact_annot = MagicMock()
|
||||
return page
|
||||
|
||||
def test_tier_0_fits_at_original_size(self, pdf_mod):
|
||||
"""If the text fits at original size + original bbox, no shrink."""
|
||||
page = self._make_page()
|
||||
block = self._make_block(font_size=12, text="Short text")
|
||||
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
result = translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
# Should have succeeded
|
||||
assert result is True
|
||||
# First call should be with original size (12) and original bbox
|
||||
first_call = page.insert_textbox.call_args_list[0]
|
||||
# First positional arg is the rect
|
||||
assert first_call[0][0] == block["bbox"]
|
||||
# Second positional arg is the text
|
||||
assert first_call[0][1] == "Short text"
|
||||
# fontsize kwarg
|
||||
assert first_call[1]["fontsize"] == 12
|
||||
|
||||
def test_tier_2_uses_expanded_vertical_bbox(self, pdf_mod):
|
||||
"""When original bbox is too small, tier 2 expands vertically."""
|
||||
page = self._make_page()
|
||||
# Force overflow on the original bbox
|
||||
page.insert_textbox = MagicMock(side_effect=[-5, -5, 0])
|
||||
# -5 = overflow, 0 = fits
|
||||
|
||||
block = self._make_block(font_size=12, x0=100, y0=100, x1=400, y1=120)
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
result = translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
assert result is True
|
||||
# Should have tried multiple times
|
||||
assert page.insert_textbox.call_count >= 3
|
||||
|
||||
def test_tier_3_4_limit_to_two_shrink_steps(self, pdf_mod):
|
||||
"""Even on persistent overflow, the shrink is limited."""
|
||||
page = self._make_page()
|
||||
# Always overflow
|
||||
page.insert_textbox = MagicMock(return_value=-5)
|
||||
|
||||
block = self._make_block(font_size=20, x0=100, y0=100, x1=400, y1=120)
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
result = translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
# Should have failed (returned False) because persistent overflow
|
||||
assert result is False
|
||||
# But the number of insert_textbox calls should be bounded
|
||||
# (not the 8+ attempts of the old aggressive strategy)
|
||||
assert page.insert_textbox.call_count <= 8
|
||||
|
||||
def test_heading_not_shrunk_below_90_percent(self, pdf_mod):
|
||||
"""A 22pt heading should never be shrunk below ~19.8pt."""
|
||||
page = self._make_page()
|
||||
call_sizes = []
|
||||
# Track every fontsize used
|
||||
def capture(*args, **kwargs):
|
||||
call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None))
|
||||
return 0 # always fit
|
||||
page.insert_textbox = MagicMock(side_effect=capture)
|
||||
|
||||
# 22pt heading
|
||||
block = self._make_block(font_size=22, text="X" * 1000)
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
# All attempted sizes should be >= 22 * 0.90 = 19.8
|
||||
for s in call_sizes:
|
||||
if s is not None:
|
||||
assert s >= 22 * 0.90 - 0.1, (
|
||||
f"Heading shrunk below 90% (got {s}, min allowed {22 * 0.90})"
|
||||
)
|
||||
|
||||
def test_body_not_shrunk_below_75_percent(self, pdf_mod):
|
||||
"""Body text (12pt) should never be shrunk below ~9pt."""
|
||||
page = self._make_page()
|
||||
call_sizes = []
|
||||
def capture(*args, **kwargs):
|
||||
call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None))
|
||||
return 0
|
||||
page.insert_textbox = MagicMock(side_effect=capture)
|
||||
|
||||
block = self._make_block(font_size=12, text="Y" * 1000)
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
for s in call_sizes:
|
||||
if s is not None:
|
||||
assert s >= 12 * 0.75 - 0.1, (
|
||||
f"Body text shrunk below 75% (got {s}, min allowed {12 * 0.75})"
|
||||
)
|
||||
|
||||
def test_graceful_failure_emits_metric(self, pdf_mod):
|
||||
"""If nothing fits, the block is skipped and a metric is recorded."""
|
||||
page = self._make_page()
|
||||
page.insert_textbox = MagicMock(return_value=-5) # always overflow
|
||||
|
||||
# Track metric emissions
|
||||
metrics_emitted = []
|
||||
def fake_record(*args, **kwargs):
|
||||
metrics_emitted.append((args, kwargs))
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
with patch.object(pdf_mod, "_record_format_loss_metric", fake_record):
|
||||
block = self._make_block(font_size=12, text="Z" * 1000)
|
||||
result = translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
assert result is False
|
||||
# A format_loss metric should have been emitted
|
||||
assert any("text_overflow" in str(call) for call in metrics_emitted), (
|
||||
f"No text_overflow metric emitted. Got: {metrics_emitted}"
|
||||
)
|
||||
|
||||
def test_successful_render_does_not_emit_loss_metric(self, pdf_mod):
|
||||
"""A block that fits cleanly should NOT trigger format_loss."""
|
||||
page = self._make_page()
|
||||
page.insert_textbox = MagicMock(return_value=0) # always fit
|
||||
|
||||
metrics_emitted = []
|
||||
def fake_record(*args, **kwargs):
|
||||
metrics_emitted.append((args, kwargs))
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
with patch.object(pdf_mod, "_record_format_loss_metric", fake_record):
|
||||
block = self._make_block(font_size=12, text="Short")
|
||||
result = translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
assert result is True
|
||||
# No format_loss should have been emitted
|
||||
assert metrics_emitted == [], (
|
||||
f"Unexpected metric emission on success: {metrics_emitted}"
|
||||
)
|
||||
|
||||
def test_min_size_floor_respected(self, pdf_mod):
|
||||
"""The min size floor (MIN_FONT_SIZE) is always respected."""
|
||||
page = self._make_page()
|
||||
call_sizes = []
|
||||
def capture(*args, **kwargs):
|
||||
call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None))
|
||||
return 0
|
||||
page.insert_textbox = MagicMock(side_effect=capture)
|
||||
|
||||
# Tiny font (3pt) — should still be at least MIN_FONT_SIZE (4.5pt)
|
||||
block = self._make_block(font_size=3, text="Q" * 1000)
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
translator._write_translated_block(
|
||||
page, block, font_path=None, is_rtl=False
|
||||
)
|
||||
# Exclude the placeholder font (used in graceful failure)
|
||||
real_sizes = [s for s in call_sizes if s is not None and s >= 4.5]
|
||||
for s in real_sizes:
|
||||
assert s >= pdf_mod.MIN_FONT_SIZE, (
|
||||
f"Font size {s} below absolute floor {pdf_mod.MIN_FONT_SIZE}"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Real PDF comparison
|
||||
# ============================================================================
|
||||
|
||||
class TestRealPDFSmartFit:
|
||||
"""Run the actual translator on a real PDF and check the output."""
|
||||
|
||||
def test_translate_real_pdf_preserves_hyperlinks(self, tmp_path):
|
||||
"""The translated PDF should keep all hyperlinks (B3 fix)."""
|
||||
import fitz
|
||||
# Build a minimal PDF in memory
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
page.insert_text((72, 72), "Visit our website for more details.")
|
||||
page.insert_link({
|
||||
"kind": fitz.LINK_URI,
|
||||
"from": fitz.Rect(72, 65, 280, 80),
|
||||
"uri": "https://example.com",
|
||||
})
|
||||
|
||||
in_path = tmp_path / "input.pdf"
|
||||
out_path = tmp_path / "output.pdf"
|
||||
doc.save(str(in_path))
|
||||
doc.close()
|
||||
|
||||
# Translate
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
translator = PDFTranslator()
|
||||
translator.translate_file(
|
||||
in_path, out_path,
|
||||
target_language="fr",
|
||||
source_language="en",
|
||||
)
|
||||
|
||||
# Verify hyperlinks preserved
|
||||
out_doc = fitz.open(str(out_path))
|
||||
out_page = out_doc[0]
|
||||
links = list(out_page.get_links())
|
||||
assert len(links) == 1
|
||||
assert links[0].get("uri") == "https://example.com"
|
||||
out_doc.close()
|
||||
|
||||
def test_translate_real_pdf_no_font_below_75pct_for_body(self, tmp_path):
|
||||
"""Body text on a real PDF should not be shrunk below 75% of original.
|
||||
|
||||
Note: the [translation overflow] placeholder font (6.0pt) is
|
||||
excluded from this check — it's a deliberate marker for skipped
|
||||
blocks, not a real text font.
|
||||
"""
|
||||
import fitz
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
# Real body text at 12pt
|
||||
page.insert_text((72, 72), "Hello world this is a test.",
|
||||
fontsize=12)
|
||||
|
||||
in_path = tmp_path / "input.pdf"
|
||||
out_path = tmp_path / "output.pdf"
|
||||
doc.save(str(in_path))
|
||||
doc.close()
|
||||
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
translator = PDFTranslator()
|
||||
translator.translate_file(
|
||||
in_path, out_path,
|
||||
target_language="fr",
|
||||
source_language="en",
|
||||
)
|
||||
|
||||
# Check output fonts — none should be below 12 * 0.75 = 9pt for body
|
||||
out_doc = fitz.open(str(out_path))
|
||||
out_page = out_doc[0]
|
||||
out_fonts = set()
|
||||
for block in out_page.get_text("dict").get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
out_fonts.add(span.get("size", 0))
|
||||
out_doc.close()
|
||||
|
||||
# Allow some shrinkage for very long text but never below 75%.
|
||||
# Exclude the placeholder font (6.0pt) which is the overflow marker.
|
||||
for size in out_fonts:
|
||||
if size > 0 and size > 6.5: # > 6.5 excludes the 6.0pt placeholder
|
||||
assert size >= 8.5, (
|
||||
f"Body text font size {size}pt is below the 9pt floor "
|
||||
f"(75% of 12pt)"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Redaction strategy
|
||||
# ============================================================================
|
||||
|
||||
class TestSingleRedactionPerBlock:
|
||||
"""Track B3.5: redaction should use a single rect per block, not per sub-bbox."""
|
||||
|
||||
def test_single_redaction_call_per_block(self, pdf_mod):
|
||||
"""The redaction loop should call add_redact_annot ONCE per block."""
|
||||
# Build a fake page
|
||||
import fitz
|
||||
page = MagicMock()
|
||||
page.rect = fitz.Rect(0, 0, 612, 792)
|
||||
redact_calls = []
|
||||
page.add_redact_annot = MagicMock(side_effect=lambda r, **k: redact_calls.append(r))
|
||||
page.apply_redactions = MagicMock()
|
||||
|
||||
# 3 blocks, each with 5 sub_bboxes
|
||||
blocks = [
|
||||
{
|
||||
"translated": "Block 1 text",
|
||||
"bbox": (50, 50, 400, 70),
|
||||
"sub_bboxes": [(50, 50, 400, 70)] * 5,
|
||||
},
|
||||
{
|
||||
"translated": "Block 2 text",
|
||||
"bbox": (50, 80, 400, 100),
|
||||
"sub_bboxes": [(50, 80, 400, 100)] * 5,
|
||||
},
|
||||
{
|
||||
"translated": "Block 3 text",
|
||||
"bbox": (50, 110, 400, 130),
|
||||
"sub_bboxes": [(50, 110, 400, 130)] * 5,
|
||||
},
|
||||
]
|
||||
|
||||
# Simulate the redaction loop from _process_pages_inplace
|
||||
for block in blocks:
|
||||
if block.get("translated"):
|
||||
page.add_redact_annot(
|
||||
fitz.Rect(block["bbox"]),
|
||||
fill=(1, 1, 1),
|
||||
)
|
||||
|
||||
# 3 blocks → 3 redaction calls (NOT 15)
|
||||
assert len(redact_calls) == 3, (
|
||||
f"Expected 3 redaction calls (one per block), got {len(redact_calls)}"
|
||||
)
|
||||
@@ -37,7 +37,19 @@ logger = get_logger(__name__)
|
||||
MIN_FONT_SIZE = 4.5
|
||||
|
||||
# Font size reduction factor when text overflows its bounding box
|
||||
FONT_SHRINK_FACTOR = 0.87
|
||||
FONT_SHRINK_FACTOR = 0.93 # Track B3.5: was 0.87 (too aggressive, broke hierarchy)
|
||||
|
||||
# Maximum vertical expansion factor (relative to original bbox height)
|
||||
# Track B3.5: was 1.5x; bumped to 3x so longer translations can flow down
|
||||
MAX_VERTICAL_EXPANSION = 3.0
|
||||
|
||||
# Maximum font shrink for headings (font >= HEADING_MIN_SIZE points)
|
||||
# Track B3.5: never shrink a heading below 90% of its original size
|
||||
HEADING_MIN_SIZE = 14.0
|
||||
HEADING_MIN_SCALE = 0.90
|
||||
|
||||
# Maximum font shrink for body text
|
||||
BODY_MIN_SCALE = 0.75
|
||||
|
||||
# RTL language codes
|
||||
RTL_LANGUAGES = frozenset({"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"})
|
||||
@@ -309,8 +321,14 @@ class PDFTranslator:
|
||||
|
||||
for block in blocks:
|
||||
if block.get("translated"):
|
||||
for sub_rect in block["sub_bboxes"]:
|
||||
page.add_redact_annot(fitz.Rect(sub_rect), fill=(1, 1, 1))
|
||||
# Track B3.5: single redaction per block, not per sub-bbox.
|
||||
# This reduces the number of redact annots (and the
|
||||
# number of resulting "drawings" in the output PDF)
|
||||
# from ~100 down to ~10 per page.
|
||||
page.add_redact_annot(
|
||||
fitz.Rect(block["bbox"]),
|
||||
fill=(1, 1, 1),
|
||||
)
|
||||
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
@@ -525,15 +543,26 @@ class PDFTranslator:
|
||||
block: Dict,
|
||||
font_path: Optional[str],
|
||||
is_rtl: bool,
|
||||
) -> None:
|
||||
) -> bool:
|
||||
"""Write translated text into the block's bounding box.
|
||||
|
||||
Priority: respect original font size as much as possible.
|
||||
Strategy:
|
||||
1. Try original rect at original font size.
|
||||
2. Expand bbox to page margins (same font size).
|
||||
3. Expand bbox vertically downward (same font size).
|
||||
4. Only THEN shrink font as a last resort, with a floor of 70% original.
|
||||
Track B3.5: Smart overflow handling. Returns True if the block
|
||||
was rendered cleanly, False if it was skipped (format loss).
|
||||
|
||||
Strategy (in order, first success wins):
|
||||
0. Try original rect at original font size.
|
||||
1. Expand bbox to page margins horizontally, original font size.
|
||||
2. Expand bbox vertically (up to MAX_VERTICAL_EXPANSION x height),
|
||||
original font size.
|
||||
3. Shrink font ONCE (FONT_SHRINK_FACTOR = 0.93), with expanded bbox.
|
||||
4. Shrink font AGAIN (0.87 cumulative), with expanded bbox.
|
||||
5. Heading: stop at HEADING_MIN_SCALE (90%). Body: BODY_MIN_SCALE (75%).
|
||||
6. If still overflow: SKIP the block, emit format_loss metric, write
|
||||
a small placeholder so the user knows something was lost.
|
||||
|
||||
Key insight: prefer LOOSING a block over DEGRADING the entire page's
|
||||
font hierarchy. A title that becomes unreadable 8.5pt is worse than
|
||||
a missing TOC entry.
|
||||
"""
|
||||
import fitz
|
||||
|
||||
@@ -544,70 +573,112 @@ class PDFTranslator:
|
||||
color = self._int_to_rgb(block["color"])
|
||||
align = fitz.TEXT_ALIGN_RIGHT if is_rtl else fitz.TEXT_ALIGN_LEFT
|
||||
|
||||
fontname = None
|
||||
# PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'.
|
||||
# If a custom font file is available, use it via fontfile (fontname ignored).
|
||||
fontname = "helv"
|
||||
fontfile = font_path
|
||||
|
||||
# Step 1: original rect, original size
|
||||
size = target_size
|
||||
rc = self._try_insert(page, original_rect, translated, size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return
|
||||
# Determine if this is a heading (larger font size = more visual weight)
|
||||
is_heading = target_size >= HEADING_MIN_SIZE
|
||||
min_scale = HEADING_MIN_SCALE if is_heading else BODY_MIN_SCALE
|
||||
min_size = max(target_size * min_scale, MIN_FONT_SIZE)
|
||||
|
||||
# Step 2: expand to page margins (horizontal)
|
||||
page_rect = page.rect
|
||||
margin = 18
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
min(original_rect.x1, page_rect.x1 - margin),
|
||||
original_rect.y1,
|
||||
)
|
||||
if expanded_h.width > original_rect.width:
|
||||
rc = self._try_insert(page, expanded_h, translated, size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return
|
||||
|
||||
# Step 3: expand vertically (allow text to flow down)
|
||||
max_expand_y = min(page_rect.y1 - margin - original_rect.y1, original_rect.height * 1.5)
|
||||
expanded = fitz.Rect(
|
||||
# Try to find a wider bbox that respects nearby blocks
|
||||
# (don't overlap with the next text block on the page)
|
||||
max_x1 = page_rect.x1 - margin
|
||||
if not is_heading:
|
||||
# For body text, allow horizontal expansion up to the page margin
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
max_x1,
|
||||
original_rect.y1,
|
||||
)
|
||||
else:
|
||||
# For headings, keep original width (preserves visual alignment)
|
||||
expanded_h = original_rect
|
||||
|
||||
# Try to find vertical room below this block
|
||||
next_block_y = page_rect.y1 - margin
|
||||
# (page.next_block_y is not exposed, so we use page bottom as ceiling)
|
||||
max_expand_y = min(
|
||||
next_block_y - original_rect.y1,
|
||||
original_rect.height * MAX_VERTICAL_EXPANSION,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0,
|
||||
expanded_h.y0,
|
||||
expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
if expanded.height > expanded_h.height:
|
||||
rc = self._try_insert(page, expanded, translated, size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return
|
||||
|
||||
# Step 4: shrink font — but never below 70% of original
|
||||
min_size = max(target_size * 0.70, MIN_FONT_SIZE)
|
||||
rect = expanded
|
||||
for attempt in range(8):
|
||||
size *= FONT_SHRINK_FACTOR
|
||||
if size < min_size:
|
||||
size = min_size
|
||||
rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align)
|
||||
break
|
||||
rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return
|
||||
# Tier 0: original rect, original size
|
||||
# insert_textbox returns >= 0 if text fit, < 0 if overflow (in points)
|
||||
rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Last resort
|
||||
if rc is None or rc < 0:
|
||||
try:
|
||||
page.insert_textbox(
|
||||
rect,
|
||||
translated,
|
||||
fontsize=min_size,
|
||||
fontname=fontname or "helv",
|
||||
fontfile=fontfile,
|
||||
color=color,
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("textbox_final_failed", error=str(e))
|
||||
# Tier 1: expanded horizontal, original size
|
||||
if expanded_h.width > original_rect.width + 1:
|
||||
rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Tier 2: expanded vertical, original size
|
||||
if expanded_v.height > original_rect.height + 1:
|
||||
rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Tier 3: shrink once
|
||||
size_1 = max(target_size * FONT_SHRINK_FACTOR, min_size)
|
||||
if size_1 < target_size - 0.1: # only try if shrink is meaningful
|
||||
rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Tier 4: shrink twice
|
||||
size_2 = max(target_size * FONT_SHRINK_FACTOR * FONT_SHRINK_FACTOR, min_size)
|
||||
if size_2 < size_1 - 0.1:
|
||||
rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Tier 5: hit the min size floor
|
||||
rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align)
|
||||
if rc is not None and rc >= 0:
|
||||
return True
|
||||
|
||||
# Tier 6: graceful failure — skip the block, log it
|
||||
self._skip_block_with_marker(page, original_rect, translated, color)
|
||||
_record_format_loss_metric("text_overflow")
|
||||
logger.warning(
|
||||
"pdf_block_skipped_overflow",
|
||||
font_size=target_size,
|
||||
text_preview=translated[:60],
|
||||
)
|
||||
return False
|
||||
|
||||
def _skip_block_with_marker(self, page, rect, original_text, color):
|
||||
"""Mark a skipped block with a visible placeholder so the user
|
||||
knows something was lost. Uses a tiny font + grey color."""
|
||||
import fitz
|
||||
placeholder = "[translation overflow]"
|
||||
try:
|
||||
page.insert_textbox(
|
||||
rect,
|
||||
placeholder,
|
||||
fontsize=6.0,
|
||||
fontname="helv",
|
||||
color=(0.6, 0.6, 0.6),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _try_insert(
|
||||
self, page, rect, text, fontsize, fontname, fontfile, color, align
|
||||
@@ -624,7 +695,16 @@ class PDFTranslator:
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
# Demote this from warning to debug — insert_textbox is called
|
||||
# 6+ times per block (one per tier), and these errors are common
|
||||
# in normal operation (e.g. when the text really doesn't fit).
|
||||
logger.debug(
|
||||
"pdf_insert_textbox_error",
|
||||
error=str(e)[:200],
|
||||
error_type=type(e).__name__,
|
||||
fontsize=fontsize,
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user