feat(format): B3.5 — PDF smart-fit rewrite + critical fontname=None fix
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
ROOT CAUSE FIX: PyMuPDF silently raised AttributeError when fontname=None was passed to insert_textbox. The try/except in _try_insert was swallowing the error and returning None, causing every block to be skipped via the graceful failure path. Setting fontname='helv' as the default unblocks the entire PDF translation pipeline. SMART-FIT: rewrite _write_translated_block with proper tier-fallback: - Tier 0: original bbox at original size - Tier 1: expanded horizontal - Tier 2: expanded vertical (3x original height) - Tier 3: shrink once (0.93x) - Tier 4: shrink twice (0.87x cumulative) - Tier 5: min size floor (90% for headings, 75% for body) - Tier 6: graceful skip with visible placeholder REDACTION: single redaction per block (was per sub-bbox, creating 100+ redaction rectangles per page). Now only 1 redaction per text block. FEATURE FLAG: PDF_SMART_FIT_ENABLED (default true, observation-first). METRICS: text_overflow -> format_elements_lost_total. RESULT ON REAL PDF: Before: fonts shrunk 22pt->5.6pt, hierarchy destroyed After: fonts EXACT match: [8, 11, 12, 14, 16, 22] preserved
This commit is contained in:
71
scripts/diagnose_pdf_layout.py
Normal file
71
scripts/diagnose_pdf_layout.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Deeper investigation: where exactly is the text on each page?
|
||||
|
||||
Compares bounding boxes of the original text vs translated text.
|
||||
If they're in totally different positions, the layout is broken.
|
||||
"""
|
||||
import fitz
|
||||
|
||||
orig = fitz.open("sample_files/test_corpus/test_pdf.pdf")
|
||||
trans = fitz.open("sample_files/test_corpus/test_pdf_translated.pdf")
|
||||
|
||||
for i in range(min(3, len(orig), len(trans))):
|
||||
o = orig[i]
|
||||
t = trans[i]
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"PAGE {i+1}")
|
||||
print('=' * 70)
|
||||
|
||||
# Get text blocks with positions for original
|
||||
print(f"\n--- Original (page {i+1}) text blocks ---")
|
||||
o_blocks = o.get_text("blocks")
|
||||
for j, b in enumerate(o_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
print(f"\n--- Translated (page {i+1}) text blocks ---")
|
||||
t_blocks = t.get_text("blocks")
|
||||
for j, b in enumerate(t_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
# Check fonts
|
||||
print(f"\n--- Fonts (page {i+1}) ---")
|
||||
o_fonts = set()
|
||||
t_fonts = set()
|
||||
for blk in o_blocks:
|
||||
for line in blk[4].split("\n") if blk[4] else []:
|
||||
pass
|
||||
# Get fonts from the dict format
|
||||
o_dict = o.get_text("dict")
|
||||
t_dict = t.get_text("dict")
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
print(f" Original fonts: {sorted(o_fonts)[:5]}")
|
||||
print(f" Translated fonts: {sorted(t_fonts)[:5]}")
|
||||
|
||||
# Check colors used
|
||||
print(f"\n--- Colors (page {i+1}) ---")
|
||||
o_colors = set()
|
||||
t_colors = set()
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_colors.add(span.get("color", 0))
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_colors.add(span.get("color", 0))
|
||||
print(f" Original colors: {sorted(o_colors)[:5]}")
|
||||
print(f" Translated colors: {sorted(t_colors)[:5]}")
|
||||
|
||||
orig.close()
|
||||
trans.close()
|
||||
Reference in New Issue
Block a user