sepehr
e706cef5d6
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
ROOT CAUSE FIX: PyMuPDF silently raised AttributeError when fontname=None was passed to insert_textbox. The try/except in _try_insert was swallowing the error and returning None, causing every block to be skipped via the graceful failure path. Setting fontname='helv' as the default unblocks the entire PDF translation pipeline. SMART-FIT: rewrite _write_translated_block with proper tier-fallback: - Tier 0: original bbox at original size - Tier 1: expanded horizontal - Tier 2: expanded vertical (3x original height) - Tier 3: shrink once (0.93x) - Tier 4: shrink twice (0.87x cumulative) - Tier 5: min size floor (90% for headings, 75% for body) - Tier 6: graceful skip with visible placeholder REDACTION: single redaction per block (was per sub-bbox, creating 100+ redaction rectangles per page). Now only 1 redaction per text block. FEATURE FLAG: PDF_SMART_FIT_ENABLED (default true, observation-first). METRICS: text_overflow -> format_elements_lost_total. RESULT ON REAL PDF: Before: fonts shrunk 22pt->5.6pt, hierarchy destroyed After: fonts EXACT match: [8, 11, 12, 14, 16, 22] preserved
108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
"""
|
|
Deep comparison of original vs translated PDF.
|
|
|
|
Outputs a detailed diff to identify exactly what format elements were lost.
|
|
"""
|
|
import fitz
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
original = Path("sample_files/test_corpus/test_pdf.pdf")
|
|
translated = Path("sample_files/test_corpus/test_pdf_translated.pdf")
|
|
|
|
print(f"Original: {original.stat().st_size:>10} bytes")
|
|
print(f"Translated: {translated.stat().st_size:>10} bytes")
|
|
print(f"Size diff: {translated.stat().st_size - original.stat().st_size:>10} bytes ({100 * translated.stat().st_size / original.stat().st_size:.1f}%)")
|
|
print()
|
|
|
|
doc_orig = fitz.open(str(original))
|
|
doc_trans = fitz.open(str(translated))
|
|
|
|
print("=" * 70)
|
|
print("PAGE COUNT")
|
|
print("=" * 70)
|
|
print(f" Original: {len(doc_orig)} pages")
|
|
print(f" Translated: {len(doc_trans)} pages")
|
|
print()
|
|
|
|
print("=" * 70)
|
|
print("PER-PAGE COMPARISON")
|
|
print("=" * 70)
|
|
for i in range(max(len(doc_orig), len(doc_trans))):
|
|
o_page = doc_orig[i] if i < len(doc_orig) else None
|
|
t_page = doc_trans[i] if i < len(doc_trans) else None
|
|
print(f"\n--- Page {i+1} ---")
|
|
if o_page:
|
|
o_text = len(o_page.get_text())
|
|
o_links = len(o_page.get_links())
|
|
o_images = len(o_page.get_images())
|
|
o_drawings = len(o_page.get_drawings())
|
|
print(f" Original: {o_text:>4} chars, {o_links:>2} links, {o_images:>2} images, {o_drawings:>3} drawings")
|
|
if t_page:
|
|
t_text = len(t_page.get_text())
|
|
t_links = len(t_page.get_links())
|
|
t_images = len(t_page.get_images())
|
|
t_drawings = len(t_page.get_drawings())
|
|
print(f" Translated: {t_text:>4} chars, {t_links:>2} links, {t_images:>2} images, {t_drawings:>3} drawings")
|
|
if o_page and t_page:
|
|
print(f" DIFF: text {t_text - o_text:+d}, links {t_links - o_links:+d}, images {t_images - o_images:+d}, drawings {t_drawings - o_drawings:+d}")
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("HYPERLINK COMPARISON")
|
|
print("=" * 70)
|
|
for i in range(max(len(doc_orig), len(doc_trans))):
|
|
o_page = doc_orig[i] if i < len(doc_orig) else None
|
|
t_page = doc_trans[i] if i < len(doc_trans) else None
|
|
o_links = o_page.get_links() if o_page else []
|
|
t_links = t_page.get_links() if t_page else []
|
|
if o_links or t_links:
|
|
print(f"\n Page {i+1}:")
|
|
print(f" Original has {len(o_links)} links:")
|
|
for link in o_links:
|
|
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
|
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
|
print(f" - {kind}: {uri_or_page[:80]}")
|
|
print(f" Translated has {len(t_links)} links:")
|
|
for link in t_links:
|
|
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
|
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
|
print(f" - {kind}: {uri_or_page[:80]}")
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("IMAGE COMPARISON")
|
|
print("=" * 70)
|
|
total_o_imgs = sum(len(p.get_images()) for p in doc_orig)
|
|
total_t_imgs = sum(len(p.get_images()) for p in doc_trans)
|
|
print(f" Original: {total_o_imgs} images")
|
|
print(f" Translated: {total_t_imgs} images")
|
|
if total_o_imgs > 0 and total_t_imgs == 0:
|
|
print(" ❌ ALL IMAGES LOST in translation!")
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("DRAWING COMPARISON (rects, lines, etc.)")
|
|
print("=" * 70)
|
|
total_o_dr = sum(len(p.get_drawings()) for p in doc_orig)
|
|
total_t_dr = sum(len(p.get_drawings()) for p in doc_trans)
|
|
print(f" Original: {total_o_dr} drawings")
|
|
print(f" Translated: {total_t_dr} drawings")
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("SAMPLE TEXT — Page 1 of translated")
|
|
print("=" * 70)
|
|
if len(doc_trans) > 0:
|
|
print(doc_trans[0].get_text()[:500])
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("SAMPLE TEXT — Page 1 of original")
|
|
print("=" * 70)
|
|
if len(doc_orig) > 0:
|
|
print(doc_orig[0].get_text()[:500])
|
|
|
|
doc_orig.close()
|
|
doc_trans.close()
|