feat(format): B3.9 — preserve PDF table column structure during translation
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m30s

User reported that the page 6 table on the test PDF ('6. Performance
and Scaling' page) was completely broken: the 3-column table
('Document size | Avg latency (s) | Throughput (docs/min)' with
5 data rows) was rendered as a vertical list of label/value pairs
instead of as a proper table.

Root cause: a PDF 'block' that contains multiple LINES at the SAME
y but different x positions is a table row (3 cells side-by-side).
The extractor was treating the whole row as one paragraph, joining
all cell texts with newline. When the smart-fit logic wrote the
text back, it used the row's full-width bbox and \insert_textbox\
wrote everything left-aligned, collapsing all columns into one.

Fix: at extraction time, detect horizontal-layout blocks (lines at
the same y, different x within 5pt tolerance) and split them into
one sub-block per line. Each cell gets its own bbox, so the
translator writes each cell at its original x position, preserving
the column structure.

Detection heuristic:
  - Block has >= 2 lines
  - All lines have y0 within 3pt of each other (SAME_ROW_Y_TOLERANCE)
  - At least 2 lines have different x0 (within > 5pt)
If all three hold, it's a table row. Otherwise, keep the old
multi-line-paragraph behavior.

Note: PyMuPDF re-groups cells into row-blocks when reading the
output back (so 'len(blocks)' looks unchanged), but the LINES
within each block are at their correct x positions. Tests check
the line x0 values, not the block count.

Visual proof: page 7 of sample_files/test_corpus/test_pdf_translated.pdf
now shows the table with proper 3-column structure (Taille du document
| Latence moyenne (s) | Débit (docs/min)) instead of an '[translation
overflow]' placeholder.

4 new tests added:
  - test_horizontal_layout_detected: 3 lines at same y -> 3 blocks
  - test_vertical_layout_kept_as_one_block: 3 lines at different y -> 1 block
  - test_single_line_block_unchanged: 1 line -> 1 block
  - test_table_cell_each_at_own_x: e2e table translation, cells at
    correct x positions

Total: 457 tests pass (was 453), zero regression.
This commit is contained in:
2026-07-14 19:25:24 +02:00
parent 4aebb49c7b
commit 2da2c4765c
3 changed files with 252 additions and 1 deletions

View File

@@ -574,3 +574,179 @@ class TestB3_8MaxExpandYNegativeClamp:
f"expected near 70"
)
out_doc.close()
# ============================================================================
# Track B3.9 — table row preservation (column structure survives translation)
# ============================================================================
class TestB3_9TableRowSplitting:
"""Track B3.9: a PDF 'block' that contains multiple LINES at the
SAME y but different x positions is a table row. The extractor must
split it into one sub-block per line so the column structure
survives translation.
Without B3.9, the whole row is treated as one paragraph and
`insert_textbox` writes the text left-aligned at x0, collapsing
all columns into one."""
def test_horizontal_layout_detected(self, pdf_mod):
"""A block with lines at the same y and different x is
detected as a horizontal layout (table row)."""
import fitz
translator = pdf_mod.PDFTranslator()
# Mock page that returns one block with 3 lines at same y
mock_page = MagicMock()
mock_page.get_text.return_value = {
"blocks": [{
"type": 0,
"bbox": (90, 188, 460, 203),
"lines": [
{"bbox": (90, 188, 162, 203), "spans": [
{"text": "Col 1", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (90, 188)},
]},
{"bbox": (220, 188, 292, 203), "spans": [
{"text": "Col 2", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (220, 188)},
]},
{"bbox": (350, 188, 460, 203), "spans": [
{"text": "Col 3", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (350, 188)},
]},
],
}],
}
blocks = translator._extract_text_blocks(mock_page)
# Should be split into 3 separate blocks (one per column)
assert len(blocks) == 3, f"Expected 3 blocks, got {len(blocks)}"
assert blocks[0]["text"] == "Col 1"
assert blocks[1]["text"] == "Col 2"
assert blocks[2]["text"] == "Col 3"
# Each block should have its own bbox (not the merged row bbox)
assert blocks[0]["bbox"][0] == 90
assert blocks[1]["bbox"][0] == 220
assert blocks[2]["bbox"][0] == 350
def test_vertical_layout_kept_as_one_block(self, pdf_mod):
"""A block with lines at DIFFERENT y (same x) is a multi-line
paragraph. It must NOT be split."""
import fitz
translator = pdf_mod.PDFTranslator()
mock_page = MagicMock()
mock_page.get_text.return_value = {
"blocks": [{
"type": 0,
"bbox": (72, 100, 540, 150),
"lines": [
{"bbox": (72, 100, 540, 115), "spans": [
{"text": "Line 1", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (72, 100)},
]},
{"bbox": (72, 120, 540, 135), "spans": [
{"text": "Line 2", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (72, 120)},
]},
{"bbox": (72, 140, 540, 150), "spans": [
{"text": "Line 3", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (72, 140)},
]},
],
}],
}
blocks = translator._extract_text_blocks(mock_page)
# Should remain 1 block with the 3 lines joined
assert len(blocks) == 1
assert blocks[0]["text"] == "Line 1\nLine 2\nLine 3"
assert blocks[0]["line_count"] == 3
def test_single_line_block_unchanged(self, pdf_mod):
"""A block with a single line is unchanged (no split)."""
import fitz
translator = pdf_mod.PDFTranslator()
mock_page = MagicMock()
mock_page.get_text.return_value = {
"blocks": [{
"type": 0,
"bbox": (72, 100, 300, 115),
"lines": [
{"bbox": (72, 100, 300, 115), "spans": [
{"text": "Just one line", "size": 12, "font": "helv",
"color": 0, "flags": 0, "origin": (72, 100)},
]},
],
}],
}
blocks = translator._extract_text_blocks(mock_page)
assert len(blocks) == 1
assert blocks[0]["text"] == "Just one line"
def test_table_cell_each_at_own_x(self, tmp_path):
"""End-to-end: a 3-column table in a PDF must be translated
with each cell staying at its original x position.
Note: PyMuPDF re-groups the cells into row-blocks when reading
the output back (3 lines per block), but each LINE (cell) must
be at its original x position. The test checks the line x0
values, not the block count.
"""
import fitz
doc = fitz.open()
page = doc.new_page(width=612, height=792)
# Build a 3-col table row by inserting text at 3 x positions on
# the same y. PyMuPDF groups them as a single block with 3 lines.
page.insert_text((90, 200), "Name", fontsize=12)
page.insert_text((220, 200), "Score", fontsize=12)
page.insert_text((350, 200), "Rank", fontsize=12)
page.insert_text((90, 220), "Alice", fontsize=12)
page.insert_text((220, 220), "95", fontsize=12)
page.insert_text((350, 220), "1", fontsize=12)
in_path = tmp_path / "input_table.pdf"
out_path = tmp_path / "output_table.pdf"
doc.save(str(in_path))
doc.close()
from translators.pdf_translator import PDFTranslator
translator = PDFTranslator()
def _flexible_translate(*args, **kwargs):
if args and hasattr(args[0], "__iter__") and not isinstance(args[0], str):
first = args[0][0] if len(args[0]) else None
if first is not None and hasattr(first, "text"):
class _R:
def __init__(self, t): self.translated_text = t
return [_R(r.text) for r in args[0]]
return list(args[0]) if args else []
mock = MagicMock()
mock.__class__.__name__ = "MagicMock"
mock.translate_batch = MagicMock(side_effect=_flexible_translate)
translator.set_provider(mock)
translator.translate_file(
in_path, out_path,
target_language="fr",
source_language="en",
)
out_doc = fitz.open(str(out_path))
out_page = out_doc[0]
text_blocks = [
b for b in out_page.get_text("dict").get("blocks", [])
if b.get("type") == 0
]
# We should see 2 row-blocks, each containing 3 cell-lines
assert len(text_blocks) == 2, (
f"Expected 2 row-blocks, got {len(text_blocks)}"
)
for row_block in text_blocks:
lines = row_block.get("lines", [])
assert len(lines) == 3, (
f"Row block should have 3 cells, got {len(lines)}"
)
# Each cell's x0 should be one of the 3 column positions
cell_x0s = sorted(line["bbox"][0] for line in lines)
assert cell_x0s == [90.0, 220.0, 350.0], (
f"Cell x0s are {cell_x0s}, expected [90, 220, 350]"
)
out_doc.close()

View File

@@ -454,12 +454,28 @@ class PDFTranslator:
return output_path
def _extract_text_blocks(self, page) -> List[Dict]:
"""Extract text blocks with position, font, and color information."""
"""Extract text blocks with position, font, and color information.
Track B3.9: when a PDF "block" contains multiple LINES at the
same y (typical of table rows where each cell is a separate line
in the block), split it into one sub-block per line. This
preserves the column structure so a translated table row stays
a table row instead of collapsing into a single left-aligned
column.
The detection is: if a block has >= 2 lines whose y0 is within
``SAME_ROW_Y_TOLERANCE`` of each other, treat it as a
multi-column row and emit one sub-block per line. Blocks whose
lines are vertically stacked (multi-line paragraphs) keep the
old behavior.
"""
import fitz
blocks = []
data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
SAME_ROW_Y_TOLERANCE = 3.0 # points
for block in data.get("blocks", []):
if block.get("type") != 0:
continue
@@ -468,6 +484,65 @@ class PDFTranslator:
if not lines:
continue
# B3.9: detect table-row pattern. A row is multiple lines
# at the same y (different x). Multi-line paragraphs have
# lines at different y (same x).
line_y0s = [line["bbox"][1] for line in lines]
same_y = all(
abs(y - line_y0s[0]) <= SAME_ROW_Y_TOLERANCE
for y in line_y0s
)
has_horizontal_layout = (
same_y and len(lines) > 1
and any(
abs(lines[i]["bbox"][0] - lines[0]["bbox"][0]) > 5
for i in range(1, len(lines))
)
)
if has_horizontal_layout:
# Multi-column row: emit one block per line.
for line in lines:
span_parts = []
spans_info = []
for span in line.get("spans", []):
text = span.get("text", "")
if text:
span_parts.append(text)
spans_info.append({
"size": span.get("size", 12),
"font": span.get("font", "Helvetica"),
"color": span.get("color", 0),
"flags": span.get("flags", 0),
"origin": span.get("origin", (0, 0)),
})
if not span_parts:
continue
line_text = "".join(span_parts).strip()
if not line_text:
continue
avg_size = (
sum(s["size"] for s in spans_info) / len(spans_info)
if spans_info else 12.0
)
first_color = spans_info[0]["color"] if spans_info else 0
is_bold = any(s["flags"] & 16 for s in spans_info)
is_italic = any(s["flags"] & 2 for s in spans_info)
blocks.append({
"bbox": tuple(line["bbox"]),
"text": line_text,
"font_size": round(avg_size, 1),
"color": first_color,
"is_bold": is_bold,
"is_italic": is_italic,
"line_count": 1,
"translated": None,
"sub_bboxes": [tuple(line["bbox"])],
"_is_table_cell": True,
})
continue
# Standard multi-line paragraph: join lines with \n.
line_parts = []
spans_info = []