feat(format): B3.9 — preserve PDF table column structure during translation
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m30s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m30s
User reported that the page 6 table on the test PDF ('6. Performance
and Scaling' page) was completely broken: the 3-column table
('Document size | Avg latency (s) | Throughput (docs/min)' with
5 data rows) was rendered as a vertical list of label/value pairs
instead of as a proper table.
Root cause: a PDF 'block' that contains multiple LINES at the SAME
y but different x positions is a table row (3 cells side-by-side).
The extractor was treating the whole row as one paragraph, joining
all cell texts with newline. When the smart-fit logic wrote the
text back, it used the row's full-width bbox and \insert_textbox\
wrote everything left-aligned, collapsing all columns into one.
Fix: at extraction time, detect horizontal-layout blocks (lines at
the same y, different x within 5pt tolerance) and split them into
one sub-block per line. Each cell gets its own bbox, so the
translator writes each cell at its original x position, preserving
the column structure.
Detection heuristic:
- Block has >= 2 lines
- All lines have y0 within 3pt of each other (SAME_ROW_Y_TOLERANCE)
- At least 2 lines have different x0 (within > 5pt)
If all three hold, it's a table row. Otherwise, keep the old
multi-line-paragraph behavior.
Note: PyMuPDF re-groups cells into row-blocks when reading the
output back (so 'len(blocks)' looks unchanged), but the LINES
within each block are at their correct x positions. Tests check
the line x0 values, not the block count.
Visual proof: page 7 of sample_files/test_corpus/test_pdf_translated.pdf
now shows the table with proper 3-column structure (Taille du document
| Latence moyenne (s) | Débit (docs/min)) instead of an '[translation
overflow]' placeholder.
4 new tests added:
- test_horizontal_layout_detected: 3 lines at same y -> 3 blocks
- test_vertical_layout_kept_as_one_block: 3 lines at different y -> 1 block
- test_single_line_block_unchanged: 1 line -> 1 block
- test_table_cell_each_at_own_x: e2e table translation, cells at
correct x positions
Total: 457 tests pass (was 453), zero regression.
This commit is contained in:
@@ -574,3 +574,179 @@ class TestB3_8MaxExpandYNegativeClamp:
|
||||
f"expected near 70"
|
||||
)
|
||||
out_doc.close()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Track B3.9 — table row preservation (column structure survives translation)
|
||||
# ============================================================================
|
||||
|
||||
class TestB3_9TableRowSplitting:
|
||||
"""Track B3.9: a PDF 'block' that contains multiple LINES at the
|
||||
SAME y but different x positions is a table row. The extractor must
|
||||
split it into one sub-block per line so the column structure
|
||||
survives translation.
|
||||
|
||||
Without B3.9, the whole row is treated as one paragraph and
|
||||
`insert_textbox` writes the text left-aligned at x0, collapsing
|
||||
all columns into one."""
|
||||
|
||||
def test_horizontal_layout_detected(self, pdf_mod):
|
||||
"""A block with lines at the same y and different x is
|
||||
detected as a horizontal layout (table row)."""
|
||||
import fitz
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
# Mock page that returns one block with 3 lines at same y
|
||||
mock_page = MagicMock()
|
||||
mock_page.get_text.return_value = {
|
||||
"blocks": [{
|
||||
"type": 0,
|
||||
"bbox": (90, 188, 460, 203),
|
||||
"lines": [
|
||||
{"bbox": (90, 188, 162, 203), "spans": [
|
||||
{"text": "Col 1", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (90, 188)},
|
||||
]},
|
||||
{"bbox": (220, 188, 292, 203), "spans": [
|
||||
{"text": "Col 2", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (220, 188)},
|
||||
]},
|
||||
{"bbox": (350, 188, 460, 203), "spans": [
|
||||
{"text": "Col 3", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (350, 188)},
|
||||
]},
|
||||
],
|
||||
}],
|
||||
}
|
||||
blocks = translator._extract_text_blocks(mock_page)
|
||||
# Should be split into 3 separate blocks (one per column)
|
||||
assert len(blocks) == 3, f"Expected 3 blocks, got {len(blocks)}"
|
||||
assert blocks[0]["text"] == "Col 1"
|
||||
assert blocks[1]["text"] == "Col 2"
|
||||
assert blocks[2]["text"] == "Col 3"
|
||||
# Each block should have its own bbox (not the merged row bbox)
|
||||
assert blocks[0]["bbox"][0] == 90
|
||||
assert blocks[1]["bbox"][0] == 220
|
||||
assert blocks[2]["bbox"][0] == 350
|
||||
|
||||
def test_vertical_layout_kept_as_one_block(self, pdf_mod):
|
||||
"""A block with lines at DIFFERENT y (same x) is a multi-line
|
||||
paragraph. It must NOT be split."""
|
||||
import fitz
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
mock_page = MagicMock()
|
||||
mock_page.get_text.return_value = {
|
||||
"blocks": [{
|
||||
"type": 0,
|
||||
"bbox": (72, 100, 540, 150),
|
||||
"lines": [
|
||||
{"bbox": (72, 100, 540, 115), "spans": [
|
||||
{"text": "Line 1", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (72, 100)},
|
||||
]},
|
||||
{"bbox": (72, 120, 540, 135), "spans": [
|
||||
{"text": "Line 2", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (72, 120)},
|
||||
]},
|
||||
{"bbox": (72, 140, 540, 150), "spans": [
|
||||
{"text": "Line 3", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (72, 140)},
|
||||
]},
|
||||
],
|
||||
}],
|
||||
}
|
||||
blocks = translator._extract_text_blocks(mock_page)
|
||||
# Should remain 1 block with the 3 lines joined
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0]["text"] == "Line 1\nLine 2\nLine 3"
|
||||
assert blocks[0]["line_count"] == 3
|
||||
|
||||
def test_single_line_block_unchanged(self, pdf_mod):
|
||||
"""A block with a single line is unchanged (no split)."""
|
||||
import fitz
|
||||
translator = pdf_mod.PDFTranslator()
|
||||
mock_page = MagicMock()
|
||||
mock_page.get_text.return_value = {
|
||||
"blocks": [{
|
||||
"type": 0,
|
||||
"bbox": (72, 100, 300, 115),
|
||||
"lines": [
|
||||
{"bbox": (72, 100, 300, 115), "spans": [
|
||||
{"text": "Just one line", "size": 12, "font": "helv",
|
||||
"color": 0, "flags": 0, "origin": (72, 100)},
|
||||
]},
|
||||
],
|
||||
}],
|
||||
}
|
||||
blocks = translator._extract_text_blocks(mock_page)
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0]["text"] == "Just one line"
|
||||
|
||||
def test_table_cell_each_at_own_x(self, tmp_path):
|
||||
"""End-to-end: a 3-column table in a PDF must be translated
|
||||
with each cell staying at its original x position.
|
||||
|
||||
Note: PyMuPDF re-groups the cells into row-blocks when reading
|
||||
the output back (3 lines per block), but each LINE (cell) must
|
||||
be at its original x position. The test checks the line x0
|
||||
values, not the block count.
|
||||
"""
|
||||
import fitz
|
||||
doc = fitz.open()
|
||||
page = doc.new_page(width=612, height=792)
|
||||
# Build a 3-col table row by inserting text at 3 x positions on
|
||||
# the same y. PyMuPDF groups them as a single block with 3 lines.
|
||||
page.insert_text((90, 200), "Name", fontsize=12)
|
||||
page.insert_text((220, 200), "Score", fontsize=12)
|
||||
page.insert_text((350, 200), "Rank", fontsize=12)
|
||||
page.insert_text((90, 220), "Alice", fontsize=12)
|
||||
page.insert_text((220, 220), "95", fontsize=12)
|
||||
page.insert_text((350, 220), "1", fontsize=12)
|
||||
|
||||
in_path = tmp_path / "input_table.pdf"
|
||||
out_path = tmp_path / "output_table.pdf"
|
||||
doc.save(str(in_path))
|
||||
doc.close()
|
||||
|
||||
from translators.pdf_translator import PDFTranslator
|
||||
translator = PDFTranslator()
|
||||
|
||||
def _flexible_translate(*args, **kwargs):
|
||||
if args and hasattr(args[0], "__iter__") and not isinstance(args[0], str):
|
||||
first = args[0][0] if len(args[0]) else None
|
||||
if first is not None and hasattr(first, "text"):
|
||||
class _R:
|
||||
def __init__(self, t): self.translated_text = t
|
||||
return [_R(r.text) for r in args[0]]
|
||||
return list(args[0]) if args else []
|
||||
|
||||
mock = MagicMock()
|
||||
mock.__class__.__name__ = "MagicMock"
|
||||
mock.translate_batch = MagicMock(side_effect=_flexible_translate)
|
||||
translator.set_provider(mock)
|
||||
translator.translate_file(
|
||||
in_path, out_path,
|
||||
target_language="fr",
|
||||
source_language="en",
|
||||
)
|
||||
|
||||
out_doc = fitz.open(str(out_path))
|
||||
out_page = out_doc[0]
|
||||
text_blocks = [
|
||||
b for b in out_page.get_text("dict").get("blocks", [])
|
||||
if b.get("type") == 0
|
||||
]
|
||||
# We should see 2 row-blocks, each containing 3 cell-lines
|
||||
assert len(text_blocks) == 2, (
|
||||
f"Expected 2 row-blocks, got {len(text_blocks)}"
|
||||
)
|
||||
for row_block in text_blocks:
|
||||
lines = row_block.get("lines", [])
|
||||
assert len(lines) == 3, (
|
||||
f"Row block should have 3 cells, got {len(lines)}"
|
||||
)
|
||||
# Each cell's x0 should be one of the 3 column positions
|
||||
cell_x0s = sorted(line["bbox"][0] for line in lines)
|
||||
assert cell_x0s == [90.0, 220.0, 350.0], (
|
||||
f"Cell x0s are {cell_x0s}, expected [90, 220, 350]"
|
||||
)
|
||||
out_doc.close()
|
||||
|
||||
Reference in New Issue
Block a user