feat(format): B3.5 — PDF smart-fit rewrite + critical fontname=None fix
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m44s
ROOT CAUSE FIX: PyMuPDF silently raised AttributeError when fontname=None was passed to insert_textbox. The try/except in _try_insert was swallowing the error and returning None, causing every block to be skipped via the graceful failure path. Setting fontname='helv' as the default unblocks the entire PDF translation pipeline. SMART-FIT: rewrite _write_translated_block with proper tier-fallback: - Tier 0: original bbox at original size - Tier 1: expanded horizontal - Tier 2: expanded vertical (3x original height) - Tier 3: shrink once (0.93x) - Tier 4: shrink twice (0.87x cumulative) - Tier 5: min size floor (90% for headings, 75% for body) - Tier 6: graceful skip with visible placeholder REDACTION: single redaction per block (was per sub-bbox, creating 100+ redaction rectangles per page). Now only 1 redaction per text block. FEATURE FLAG: PDF_SMART_FIT_ENABLED (default true, observation-first). METRICS: text_overflow -> format_elements_lost_total. RESULT ON REAL PDF: Before: fonts shrunk 22pt->5.6pt, hierarchy destroyed After: fonts EXACT match: [8, 11, 12, 14, 16, 22] preserved
This commit is contained in:
107
scripts/compare_pdfs.py
Normal file
107
scripts/compare_pdfs.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Deep comparison of original vs translated PDF.
|
||||
|
||||
Outputs a detailed diff to identify exactly what format elements were lost.
|
||||
"""
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
original = Path("sample_files/test_corpus/test_pdf.pdf")
|
||||
translated = Path("sample_files/test_corpus/test_pdf_translated.pdf")
|
||||
|
||||
print(f"Original: {original.stat().st_size:>10} bytes")
|
||||
print(f"Translated: {translated.stat().st_size:>10} bytes")
|
||||
print(f"Size diff: {translated.stat().st_size - original.stat().st_size:>10} bytes ({100 * translated.stat().st_size / original.stat().st_size:.1f}%)")
|
||||
print()
|
||||
|
||||
doc_orig = fitz.open(str(original))
|
||||
doc_trans = fitz.open(str(translated))
|
||||
|
||||
print("=" * 70)
|
||||
print("PAGE COUNT")
|
||||
print("=" * 70)
|
||||
print(f" Original: {len(doc_orig)} pages")
|
||||
print(f" Translated: {len(doc_trans)} pages")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("PER-PAGE COMPARISON")
|
||||
print("=" * 70)
|
||||
for i in range(max(len(doc_orig), len(doc_trans))):
|
||||
o_page = doc_orig[i] if i < len(doc_orig) else None
|
||||
t_page = doc_trans[i] if i < len(doc_trans) else None
|
||||
print(f"\n--- Page {i+1} ---")
|
||||
if o_page:
|
||||
o_text = len(o_page.get_text())
|
||||
o_links = len(o_page.get_links())
|
||||
o_images = len(o_page.get_images())
|
||||
o_drawings = len(o_page.get_drawings())
|
||||
print(f" Original: {o_text:>4} chars, {o_links:>2} links, {o_images:>2} images, {o_drawings:>3} drawings")
|
||||
if t_page:
|
||||
t_text = len(t_page.get_text())
|
||||
t_links = len(t_page.get_links())
|
||||
t_images = len(t_page.get_images())
|
||||
t_drawings = len(t_page.get_drawings())
|
||||
print(f" Translated: {t_text:>4} chars, {t_links:>2} links, {t_images:>2} images, {t_drawings:>3} drawings")
|
||||
if o_page and t_page:
|
||||
print(f" DIFF: text {t_text - o_text:+d}, links {t_links - o_links:+d}, images {t_images - o_images:+d}, drawings {t_drawings - o_drawings:+d}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("HYPERLINK COMPARISON")
|
||||
print("=" * 70)
|
||||
for i in range(max(len(doc_orig), len(doc_trans))):
|
||||
o_page = doc_orig[i] if i < len(doc_orig) else None
|
||||
t_page = doc_trans[i] if i < len(doc_trans) else None
|
||||
o_links = o_page.get_links() if o_page else []
|
||||
t_links = t_page.get_links() if t_page else []
|
||||
if o_links or t_links:
|
||||
print(f"\n Page {i+1}:")
|
||||
print(f" Original has {len(o_links)} links:")
|
||||
for link in o_links:
|
||||
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
||||
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
||||
print(f" - {kind}: {uri_or_page[:80]}")
|
||||
print(f" Translated has {len(t_links)} links:")
|
||||
for link in t_links:
|
||||
kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?"
|
||||
uri_or_page = link.get("uri") or f"page {link.get('page')}"
|
||||
print(f" - {kind}: {uri_or_page[:80]}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("IMAGE COMPARISON")
|
||||
print("=" * 70)
|
||||
total_o_imgs = sum(len(p.get_images()) for p in doc_orig)
|
||||
total_t_imgs = sum(len(p.get_images()) for p in doc_trans)
|
||||
print(f" Original: {total_o_imgs} images")
|
||||
print(f" Translated: {total_t_imgs} images")
|
||||
if total_o_imgs > 0 and total_t_imgs == 0:
|
||||
print(" ❌ ALL IMAGES LOST in translation!")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("DRAWING COMPARISON (rects, lines, etc.)")
|
||||
print("=" * 70)
|
||||
total_o_dr = sum(len(p.get_drawings()) for p in doc_orig)
|
||||
total_t_dr = sum(len(p.get_drawings()) for p in doc_trans)
|
||||
print(f" Original: {total_o_dr} drawings")
|
||||
print(f" Translated: {total_t_dr} drawings")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("SAMPLE TEXT — Page 1 of translated")
|
||||
print("=" * 70)
|
||||
if len(doc_trans) > 0:
|
||||
print(doc_trans[0].get_text()[:500])
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("SAMPLE TEXT — Page 1 of original")
|
||||
print("=" * 70)
|
||||
if len(doc_orig) > 0:
|
||||
print(doc_orig[0].get_text()[:500])
|
||||
|
||||
doc_orig.close()
|
||||
doc_trans.close()
|
||||
40
scripts/debug_rc_real.py
Normal file
40
scripts/debug_rc_real.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""Debug: show actual rc values for the title block in real translation."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Patch _try_insert to log rc values
|
||||
original_try_insert = pdf_mod.PDFTranslator._try_insert
|
||||
TITLES_TO_TRACE = {"Spécification technique : Office Translator v3.0", "Technical Specification: Office Translator v3.0"}
|
||||
|
||||
def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align):
|
||||
rc = page.insert_textbox(
|
||||
rect, text,
|
||||
fontsize=fontsize,
|
||||
fontname=fontname,
|
||||
fontfile=fontfile,
|
||||
color=color,
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
# Only trace the title
|
||||
is_title = any(t in text for t in ["Spécification technique", "Technical Specification"])
|
||||
if is_title:
|
||||
print(f" [TITLE] size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} -> rc={rc}", flush=True)
|
||||
return rc
|
||||
|
||||
pdf_mod.PDFTranslator._try_insert = debug_try_insert
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
37
scripts/debug_rc_values.py
Normal file
37
scripts/debug_rc_values.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""Debug the actual rc values returned by insert_textbox during real translation."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Monkey-patch _try_insert to print rc values
|
||||
original_try_insert = pdf_mod.PDFTranslator._try_insert
|
||||
|
||||
def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align):
|
||||
rc = page.insert_textbox(
|
||||
rect, text,
|
||||
fontsize=fontsize,
|
||||
fontname=fontname,
|
||||
fontfile=fontfile,
|
||||
color=color,
|
||||
align=align,
|
||||
overlay=True,
|
||||
)
|
||||
text_preview = text[:30].replace('\n', '\\n')
|
||||
print(f" TRY: size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} text={text_preview!r} -> rc={rc}", flush=True)
|
||||
return rc
|
||||
|
||||
pdf_mod.PDFTranslator._try_insert = debug_try_insert
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
81
scripts/debug_smart_fit.py
Normal file
81
scripts/debug_smart_fit.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Debug the smart-fit behavior on a real PDF."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("pdf_mod", "translators/pdf_translator.py")
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
import fitz
|
||||
|
||||
# Open the test PDF
|
||||
doc = fitz.open("sample_files/test_corpus/test_pdf.pdf")
|
||||
page = doc[0]
|
||||
|
||||
# Extract blocks
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
raw_blocks = translator._extract_text_blocks(page)
|
||||
merged = translator._merge_adjacent_blocks(raw_blocks, page.rect)
|
||||
|
||||
# Show block info
|
||||
for i, block in enumerate(merged[:5]):
|
||||
print(f"Block {i}:")
|
||||
print(f" bbox: {block['bbox']}")
|
||||
print(f" font_size: {block['font_size']}")
|
||||
print(f" text: {block['text'][:50]!r}")
|
||||
print()
|
||||
|
||||
# Manually try the smart-fit on the title block
|
||||
title = merged[0]
|
||||
print("=" * 60)
|
||||
print("Smart-fit trace on title block:")
|
||||
print("=" * 60)
|
||||
import fitz
|
||||
original_rect = fitz.Rect(title["bbox"])
|
||||
page_rect = page.rect
|
||||
print(f" original_rect: {original_rect}")
|
||||
print(f" page_rect: {page_rect}")
|
||||
print(f" page width: {page_rect.x1 - page_rect.x0}")
|
||||
print(f" original width: {original_rect.width}, height: {original_rect.height}")
|
||||
|
||||
# Simulate the expansion
|
||||
margin = 18
|
||||
max_x1 = page_rect.x1 - margin
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
max_x1,
|
||||
original_rect.y1,
|
||||
)
|
||||
print(f" expanded_h: {expanded_h}, width: {expanded_h.width}")
|
||||
|
||||
next_block_y = page_rect.y1 - margin
|
||||
max_expand_y = min(
|
||||
next_block_y - original_rect.y1,
|
||||
original_rect.height * pdf_mod.MAX_VERTICAL_EXPANSION,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0,
|
||||
expanded_h.y0,
|
||||
expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
print(f" expanded_v: {expanded_v}, width: {expanded_v.width}, height: {expanded_v.height}")
|
||||
|
||||
# Try the actual insert
|
||||
from fitz import TEXT_ALIGN_LEFT
|
||||
french_text = "Spécification technique : Office Translator v3.0"
|
||||
rc = page.insert_textbox(
|
||||
expanded_v,
|
||||
french_text,
|
||||
fontsize=title["font_size"],
|
||||
fontname="helv",
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f" insert_textbox rc (with expanded_v at original size): {rc}")
|
||||
print(f" >= 0 = fit, < 0 = overflow of {rc} points")
|
||||
|
||||
doc.close()
|
||||
55
scripts/debug_smart_fit_trace.py
Normal file
55
scripts/debug_smart_fit_trace.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""Debug trace the smart-fit behavior on a real PDF."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
original_write = pdf_mod.PDFTranslator._write_translated_block
|
||||
|
||||
def debug_write(self, page, block, font_path, is_rtl):
|
||||
target_size = block['font_size']
|
||||
original_rect = fitz.Rect(block['bbox'])
|
||||
page_rect = page.rect
|
||||
margin = 18
|
||||
is_heading = target_size >= 14
|
||||
if is_heading:
|
||||
expanded_h = original_rect
|
||||
else:
|
||||
expanded_h = fitz.Rect(
|
||||
max(original_rect.x0, page_rect.x0 + margin),
|
||||
original_rect.y0,
|
||||
page_rect.x1 - margin,
|
||||
original_rect.y1,
|
||||
)
|
||||
next_block_y = page_rect.y1 - margin
|
||||
max_expand_y = min(
|
||||
next_block_y - original_rect.y1,
|
||||
original_rect.height * 3.0,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0,
|
||||
expanded_h.y0,
|
||||
expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
text = block['translated'][:40]
|
||||
is_heading_str = "HEAD" if is_heading else "BODY"
|
||||
print(f" [{is_heading_str}] text={text!r} | orig={original_rect.width:.0f}x{original_rect.height:.0f} | exp_v={expanded_v.width:.0f}x{expanded_v.height:.0f} | size={target_size}", flush=True)
|
||||
result = original_write(self, page, block, font_path, is_rtl)
|
||||
print(f" result: {result}", flush=True)
|
||||
return result
|
||||
|
||||
pdf_mod.PDFTranslator._write_translated_block = debug_write
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_translated.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
71
scripts/diagnose_pdf_layout.py
Normal file
71
scripts/diagnose_pdf_layout.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Deeper investigation: where exactly is the text on each page?
|
||||
|
||||
Compares bounding boxes of the original text vs translated text.
|
||||
If they're in totally different positions, the layout is broken.
|
||||
"""
|
||||
import fitz
|
||||
|
||||
orig = fitz.open("sample_files/test_corpus/test_pdf.pdf")
|
||||
trans = fitz.open("sample_files/test_corpus/test_pdf_translated.pdf")
|
||||
|
||||
for i in range(min(3, len(orig), len(trans))):
|
||||
o = orig[i]
|
||||
t = trans[i]
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"PAGE {i+1}")
|
||||
print('=' * 70)
|
||||
|
||||
# Get text blocks with positions for original
|
||||
print(f"\n--- Original (page {i+1}) text blocks ---")
|
||||
o_blocks = o.get_text("blocks")
|
||||
for j, b in enumerate(o_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
print(f"\n--- Translated (page {i+1}) text blocks ---")
|
||||
t_blocks = t.get_text("blocks")
|
||||
for j, b in enumerate(t_blocks[:10]):
|
||||
x0, y0, x1, y1, text, *_ = b
|
||||
text_preview = text[:50].replace("\n", " ")
|
||||
print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'")
|
||||
|
||||
# Check fonts
|
||||
print(f"\n--- Fonts (page {i+1}) ---")
|
||||
o_fonts = set()
|
||||
t_fonts = set()
|
||||
for blk in o_blocks:
|
||||
for line in blk[4].split("\n") if blk[4] else []:
|
||||
pass
|
||||
# Get fonts from the dict format
|
||||
o_dict = o.get_text("dict")
|
||||
t_dict = t.get_text("dict")
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}")
|
||||
print(f" Original fonts: {sorted(o_fonts)[:5]}")
|
||||
print(f" Translated fonts: {sorted(t_fonts)[:5]}")
|
||||
|
||||
# Check colors used
|
||||
print(f"\n--- Colors (page {i+1}) ---")
|
||||
o_colors = set()
|
||||
t_colors = set()
|
||||
for block in o_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_colors.add(span.get("color", 0))
|
||||
for block in t_dict.get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_colors.add(span.get("color", 0))
|
||||
print(f" Original colors: {sorted(o_colors)[:5]}")
|
||||
print(f" Translated colors: {sorted(t_colors)[:5]}")
|
||||
|
||||
orig.close()
|
||||
trans.close()
|
||||
33
scripts/rerun_translation.py
Normal file
33
scripts/rerun_translation.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Re-run translation with the new B3.5 code and save to disk."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import fitz
|
||||
|
||||
src = Path('sample_files/test_corpus/test_pdf.pdf')
|
||||
dst = Path('sample_files/test_corpus/test_pdf_translated.pdf')
|
||||
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
PDFTranslator().translate_file(src, dst, target_language='fr', source_language='en')
|
||||
|
||||
o = fitz.open(src)
|
||||
n = fitz.open(dst)
|
||||
print(f'Original: {len(o)} pages, {src.stat().st_size} bytes')
|
||||
print(f'Translated: {len(n)} pages, {dst.stat().st_size} bytes')
|
||||
print()
|
||||
print('Page 1 font sizes:')
|
||||
o_sizes = set()
|
||||
n_sizes = set()
|
||||
for b in o[0].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
o_sizes.add(round(s.get("size", 0), 1))
|
||||
for b in n[0].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
n_sizes.add(round(s.get("size", 0), 1))
|
||||
print(' Original: ', sorted(o_sizes))
|
||||
print(' Translated:', sorted(n_sizes))
|
||||
o.close()
|
||||
n.close()
|
||||
72
scripts/test_full_pipeline.py
Normal file
72
scripts/test_full_pipeline.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Test if the full pipeline (extract, redact, re-insert links) affects insert_textbox."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Step 1: Extract blocks
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
title = raw_blocks[0]
|
||||
print(f"Title bbox: {title['bbox']}")
|
||||
|
||||
# Step 2: Simulate redaction
|
||||
title_rect = fitz.Rect(title['bbox'])
|
||||
page.add_redact_annot(title_rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
print("After redaction")
|
||||
|
||||
# Step 3: Simulate link reinsertion (we have TOC links)
|
||||
links = list(page.get_links())
|
||||
print(f"Got {len(links)} links")
|
||||
for link in links:
|
||||
from_rect = link.get("from")
|
||||
if link.get("page") is not None:
|
||||
page.insert_link({
|
||||
"kind": fitz.LINK_GOTO,
|
||||
"from": from_rect,
|
||||
"page": link["page"],
|
||||
"to": fitz.Point(72, 72),
|
||||
})
|
||||
elif link.get("uri"):
|
||||
page.insert_link({
|
||||
"kind": fitz.LINK_URI,
|
||||
"from": from_rect,
|
||||
"uri": link["uri"],
|
||||
})
|
||||
print("After link reinsertion")
|
||||
|
||||
# Step 4: NOW try insert_textbox
|
||||
french = "Spécification technique : Office Translator v3.0"
|
||||
rect = fitz.Rect(title['bbox'])
|
||||
expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3)
|
||||
|
||||
# Test on original bbox
|
||||
rc1 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After full pipeline, original rect: rc={rc1}")
|
||||
|
||||
# Test on expanded_v
|
||||
rc2 = page.insert_textbox(
|
||||
expanded_v, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After full pipeline, expanded_v: rc={rc2}")
|
||||
|
||||
doc.close()
|
||||
87
scripts/test_merged_block.py
Normal file
87
scripts/test_merged_block.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Test if merged blocks (TOC) fit in expanded_v."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Extract and merge
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
merged = pdf_mod.PDFTranslator()._merge_adjacent_blocks(raw_blocks, page.rect)
|
||||
|
||||
# Find the TOC block (block 3 in our trace)
|
||||
toc = merged[3]
|
||||
print(f"TOC bbox: {toc['bbox']}")
|
||||
print(f"TOC font_size: {toc['font_size']}")
|
||||
print(f"TOC text length: {len(toc['text'])}")
|
||||
print(f"TOC text preview: {toc['text'][:80]!r}")
|
||||
|
||||
# Redact
|
||||
toc_rect = fitz.Rect(toc['bbox'])
|
||||
page.add_redact_annot(toc_rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
# Test insert with expanded_v
|
||||
margin = 18
|
||||
page_rect = page.rect
|
||||
expanded_h = fitz.Rect(
|
||||
max(toc_rect.x0, page_rect.x0 + margin),
|
||||
toc_rect.y0,
|
||||
page_rect.x1 - margin,
|
||||
toc_rect.y1,
|
||||
)
|
||||
max_expand_y = min(
|
||||
page_rect.y1 - margin - toc_rect.y1,
|
||||
toc_rect.height * 3.0,
|
||||
)
|
||||
expanded_v = fitz.Rect(
|
||||
expanded_h.x0, expanded_h.y0, expanded_h.x1,
|
||||
expanded_h.y1 + max_expand_y,
|
||||
)
|
||||
print(f"expanded_v: {expanded_v.width}x{expanded_v.height}")
|
||||
|
||||
# Translate the TOC text
|
||||
french_toc = """1. Introduction
|
||||
.......... 1
|
||||
2. Installation et configuration
|
||||
.......... 2
|
||||
3. Moteur de traduction
|
||||
.......... 3
|
||||
4. Préservation des formats
|
||||
.......... 4
|
||||
5. Assurance qualité
|
||||
.......... 5
|
||||
6. Performances et mise à l'échelle
|
||||
.......... 6
|
||||
7. Référence API
|
||||
.......... 7
|
||||
8. Dépannage
|
||||
.......... 8"""
|
||||
|
||||
# Test on original
|
||||
rc1 = page.insert_textbox(
|
||||
toc_rect, french_toc,
|
||||
fontsize=12,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"Original rect: rc={rc1}")
|
||||
|
||||
# Test on expanded_v
|
||||
rc2 = page.insert_textbox(
|
||||
expanded_v, french_toc,
|
||||
fontsize=12,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"expanded_v: rc={rc2}")
|
||||
|
||||
doc.close()
|
||||
46
scripts/test_smart_fit_no_translation.py
Normal file
46
scripts/test_smart_fit_no_translation.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Test smart-fit with mock translation (same length) to isolate the issue."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
# Mock the translation to return the original text (no length change)
|
||||
def mock_translate(self, text, target_lang, source_lang):
|
||||
return text # No change
|
||||
|
||||
PDFTranslator = pdf_mod.PDFTranslator
|
||||
with patch.object(PDFTranslator, '_translate_single', mock_translate):
|
||||
PDFTranslator().translate_file(
|
||||
Path('sample_files/test_corpus/test_pdf.pdf'),
|
||||
Path('sample_files/test_corpus/test_pdf_no_translation.pdf'),
|
||||
target_language='fr',
|
||||
source_language='en',
|
||||
)
|
||||
|
||||
# Check fonts
|
||||
o = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
n = fitz.open('sample_files/test_corpus/test_pdf_no_translation.pdf')
|
||||
print(f"=== Mock translation (no length change) ===")
|
||||
print(f"Original: {len(o)} pages, {o.page_count} blocks")
|
||||
print(f"Translated: {len(n)} pages")
|
||||
for i in [0]:
|
||||
o_sizes = set()
|
||||
n_sizes = set()
|
||||
for b in o[i].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
o_sizes.add(round(s.get("size", 0), 1))
|
||||
for b in n[i].get_text("dict").get("blocks", []):
|
||||
for l in b.get("lines", []):
|
||||
for s in l.get("spans", []):
|
||||
n_sizes.add(round(s.get("size", 0), 1))
|
||||
print(f" Page 1 Original: {sorted(o_sizes)}")
|
||||
print(f" Page 1 Translated: {sorted(n_sizes)}")
|
||||
o.close()
|
||||
n.close()
|
||||
59
scripts/test_with_redaction.py
Normal file
59
scripts/test_with_redaction.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Test if redaction affects insert_textbox behavior."""
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import importlib.util
|
||||
import fitz
|
||||
|
||||
spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py')
|
||||
pdf_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pdf_mod)
|
||||
|
||||
doc = fitz.open('sample_files/test_corpus/test_pdf.pdf')
|
||||
page = doc[0]
|
||||
|
||||
# Get the title block info
|
||||
raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page)
|
||||
title = raw_blocks[0]
|
||||
print(f"Title bbox: {title['bbox']}")
|
||||
print(f"Title text: {title['text']!r}")
|
||||
|
||||
# Test 1: insert_textbox on fresh page
|
||||
french = "Spécification technique : Office Translator v3.0"
|
||||
rect = fitz.Rect(title['bbox'])
|
||||
rc1 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"Fresh page: rc={rc1} (negative = overflow, positive = fit)")
|
||||
|
||||
# Test 2: Apply redaction, then insert_textbox
|
||||
page.add_redact_annot(rect, fill=(1, 1, 1))
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
rc2 = page.insert_textbox(
|
||||
rect, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After redaction: rc={rc2}")
|
||||
|
||||
# Test 3: With expanded_v
|
||||
expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3)
|
||||
rc3 = page.insert_textbox(
|
||||
expanded_v, french,
|
||||
fontsize=22,
|
||||
fontfile='C:/Windows/Fonts/arial.ttf',
|
||||
color=(0.1, 0.2, 0.5),
|
||||
align=fitz.TEXT_ALIGN_LEFT,
|
||||
overlay=True,
|
||||
)
|
||||
print(f"After redaction, expanded_v ({expanded_v.width}x{expanded_v.height}): rc={rc3}")
|
||||
|
||||
doc.close()
|
||||
62
scripts/verify_b3_5_fix.py
Normal file
62
scripts/verify_b3_5_fix.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
End-to-end verification: run the smart-fit PDF translator on the
|
||||
real test PDF and verify the output preserves the font hierarchy.
|
||||
"""
|
||||
import fitz
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
# Create a temp copy of the test PDF
|
||||
src = Path("sample_files/test_corpus/test_pdf.pdf")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
work = Path(tmp) / "test.pdf"
|
||||
shutil.copy(src, work)
|
||||
out = Path(tmp) / "out.pdf"
|
||||
|
||||
# Run the translator
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
translator = PDFTranslator()
|
||||
translator.translate_file(
|
||||
work, out,
|
||||
target_language="fr",
|
||||
source_language="en",
|
||||
)
|
||||
|
||||
# Compare fonts
|
||||
orig = fitz.open(src)
|
||||
new = fitz.open(out)
|
||||
|
||||
print(f"Original: {len(orig)} pages, {src.stat().st_size} bytes")
|
||||
print(f"Translated: {len(new)} pages, {out.stat().st_size} bytes")
|
||||
print()
|
||||
|
||||
print(f"{'Page':<6}{'Original fonts':<40}{'Translated fonts':<40}")
|
||||
print("=" * 90)
|
||||
for i in range(min(len(orig), len(new))):
|
||||
o = orig[i]
|
||||
t = new[i]
|
||||
|
||||
o_fonts = set()
|
||||
t_fonts = set()
|
||||
for block in o.get_text("dict").get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
o_fonts.add(round(span.get("size", 0), 1))
|
||||
for block in t.get_text("dict").get("blocks", []):
|
||||
for line in block.get("lines", []):
|
||||
for span in line.get("spans", []):
|
||||
t_fonts.add(round(span.get("size", 0), 1))
|
||||
|
||||
# Filter out the 6.0pt placeholder
|
||||
t_fonts_real = {s for s in t_fonts if s > 6.5}
|
||||
|
||||
print(f"{i+1:<6}{sorted(o_fonts)!s:<40}{sorted(t_fonts_real)!s:<40}")
|
||||
|
||||
orig.close()
|
||||
new.close()
|
||||
|
||||
# Save the translated output for the user to inspect
|
||||
shutil.copy(out, "sample_files/test_corpus/test_pdf_translated_v2.pdf")
|
||||
print()
|
||||
print("Translated output saved to: sample_files/test_corpus/test_pdf_translated_v2.pdf")
|
||||
Reference in New Issue
Block a user