feat(format): B3.9 — preserve PDF table column structure during translation
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m30s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m30s
User reported that the page 6 table on the test PDF ('6. Performance
and Scaling' page) was completely broken: the 3-column table
('Document size | Avg latency (s) | Throughput (docs/min)' with
5 data rows) was rendered as a vertical list of label/value pairs
instead of as a proper table.
Root cause: a PDF 'block' that contains multiple LINES at the SAME
y but different x positions is a table row (3 cells side-by-side).
The extractor was treating the whole row as one paragraph, joining
all cell texts with newline. When the smart-fit logic wrote the
text back, it used the row's full-width bbox and \insert_textbox\
wrote everything left-aligned, collapsing all columns into one.
Fix: at extraction time, detect horizontal-layout blocks (lines at
the same y, different x within 5pt tolerance) and split them into
one sub-block per line. Each cell gets its own bbox, so the
translator writes each cell at its original x position, preserving
the column structure.
Detection heuristic:
- Block has >= 2 lines
- All lines have y0 within 3pt of each other (SAME_ROW_Y_TOLERANCE)
- At least 2 lines have different x0 (within > 5pt)
If all three hold, it's a table row. Otherwise, keep the old
multi-line-paragraph behavior.
Note: PyMuPDF re-groups cells into row-blocks when reading the
output back (so 'len(blocks)' looks unchanged), but the LINES
within each block are at their correct x positions. Tests check
the line x0 values, not the block count.
Visual proof: page 7 of sample_files/test_corpus/test_pdf_translated.pdf
now shows the table with proper 3-column structure (Taille du document
| Latence moyenne (s) | Débit (docs/min)) instead of an '[translation
overflow]' placeholder.
4 new tests added:
- test_horizontal_layout_detected: 3 lines at same y -> 3 blocks
- test_vertical_layout_kept_as_one_block: 3 lines at different y -> 1 block
- test_single_line_block_unchanged: 1 line -> 1 block
- test_table_cell_each_at_own_x: e2e table translation, cells at
correct x positions
Total: 457 tests pass (was 453), zero regression.
This commit is contained in:
Binary file not shown.
@@ -574,3 +574,179 @@ class TestB3_8MaxExpandYNegativeClamp:
|
|||||||
f"expected near 70"
|
f"expected near 70"
|
||||||
)
|
)
|
||||||
out_doc.close()
|
out_doc.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Track B3.9 — table row preservation (column structure survives translation)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
class TestB3_9TableRowSplitting:
|
||||||
|
"""Track B3.9: a PDF 'block' that contains multiple LINES at the
|
||||||
|
SAME y but different x positions is a table row. The extractor must
|
||||||
|
split it into one sub-block per line so the column structure
|
||||||
|
survives translation.
|
||||||
|
|
||||||
|
Without B3.9, the whole row is treated as one paragraph and
|
||||||
|
`insert_textbox` writes the text left-aligned at x0, collapsing
|
||||||
|
all columns into one."""
|
||||||
|
|
||||||
|
def test_horizontal_layout_detected(self, pdf_mod):
|
||||||
|
"""A block with lines at the same y and different x is
|
||||||
|
detected as a horizontal layout (table row)."""
|
||||||
|
import fitz
|
||||||
|
translator = pdf_mod.PDFTranslator()
|
||||||
|
# Mock page that returns one block with 3 lines at same y
|
||||||
|
mock_page = MagicMock()
|
||||||
|
mock_page.get_text.return_value = {
|
||||||
|
"blocks": [{
|
||||||
|
"type": 0,
|
||||||
|
"bbox": (90, 188, 460, 203),
|
||||||
|
"lines": [
|
||||||
|
{"bbox": (90, 188, 162, 203), "spans": [
|
||||||
|
{"text": "Col 1", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (90, 188)},
|
||||||
|
]},
|
||||||
|
{"bbox": (220, 188, 292, 203), "spans": [
|
||||||
|
{"text": "Col 2", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (220, 188)},
|
||||||
|
]},
|
||||||
|
{"bbox": (350, 188, 460, 203), "spans": [
|
||||||
|
{"text": "Col 3", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (350, 188)},
|
||||||
|
]},
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
blocks = translator._extract_text_blocks(mock_page)
|
||||||
|
# Should be split into 3 separate blocks (one per column)
|
||||||
|
assert len(blocks) == 3, f"Expected 3 blocks, got {len(blocks)}"
|
||||||
|
assert blocks[0]["text"] == "Col 1"
|
||||||
|
assert blocks[1]["text"] == "Col 2"
|
||||||
|
assert blocks[2]["text"] == "Col 3"
|
||||||
|
# Each block should have its own bbox (not the merged row bbox)
|
||||||
|
assert blocks[0]["bbox"][0] == 90
|
||||||
|
assert blocks[1]["bbox"][0] == 220
|
||||||
|
assert blocks[2]["bbox"][0] == 350
|
||||||
|
|
||||||
|
def test_vertical_layout_kept_as_one_block(self, pdf_mod):
|
||||||
|
"""A block with lines at DIFFERENT y (same x) is a multi-line
|
||||||
|
paragraph. It must NOT be split."""
|
||||||
|
import fitz
|
||||||
|
translator = pdf_mod.PDFTranslator()
|
||||||
|
mock_page = MagicMock()
|
||||||
|
mock_page.get_text.return_value = {
|
||||||
|
"blocks": [{
|
||||||
|
"type": 0,
|
||||||
|
"bbox": (72, 100, 540, 150),
|
||||||
|
"lines": [
|
||||||
|
{"bbox": (72, 100, 540, 115), "spans": [
|
||||||
|
{"text": "Line 1", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (72, 100)},
|
||||||
|
]},
|
||||||
|
{"bbox": (72, 120, 540, 135), "spans": [
|
||||||
|
{"text": "Line 2", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (72, 120)},
|
||||||
|
]},
|
||||||
|
{"bbox": (72, 140, 540, 150), "spans": [
|
||||||
|
{"text": "Line 3", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (72, 140)},
|
||||||
|
]},
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
blocks = translator._extract_text_blocks(mock_page)
|
||||||
|
# Should remain 1 block with the 3 lines joined
|
||||||
|
assert len(blocks) == 1
|
||||||
|
assert blocks[0]["text"] == "Line 1\nLine 2\nLine 3"
|
||||||
|
assert blocks[0]["line_count"] == 3
|
||||||
|
|
||||||
|
def test_single_line_block_unchanged(self, pdf_mod):
|
||||||
|
"""A block with a single line is unchanged (no split)."""
|
||||||
|
import fitz
|
||||||
|
translator = pdf_mod.PDFTranslator()
|
||||||
|
mock_page = MagicMock()
|
||||||
|
mock_page.get_text.return_value = {
|
||||||
|
"blocks": [{
|
||||||
|
"type": 0,
|
||||||
|
"bbox": (72, 100, 300, 115),
|
||||||
|
"lines": [
|
||||||
|
{"bbox": (72, 100, 300, 115), "spans": [
|
||||||
|
{"text": "Just one line", "size": 12, "font": "helv",
|
||||||
|
"color": 0, "flags": 0, "origin": (72, 100)},
|
||||||
|
]},
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
blocks = translator._extract_text_blocks(mock_page)
|
||||||
|
assert len(blocks) == 1
|
||||||
|
assert blocks[0]["text"] == "Just one line"
|
||||||
|
|
||||||
|
def test_table_cell_each_at_own_x(self, tmp_path):
|
||||||
|
"""End-to-end: a 3-column table in a PDF must be translated
|
||||||
|
with each cell staying at its original x position.
|
||||||
|
|
||||||
|
Note: PyMuPDF re-groups the cells into row-blocks when reading
|
||||||
|
the output back (3 lines per block), but each LINE (cell) must
|
||||||
|
be at its original x position. The test checks the line x0
|
||||||
|
values, not the block count.
|
||||||
|
"""
|
||||||
|
import fitz
|
||||||
|
doc = fitz.open()
|
||||||
|
page = doc.new_page(width=612, height=792)
|
||||||
|
# Build a 3-col table row by inserting text at 3 x positions on
|
||||||
|
# the same y. PyMuPDF groups them as a single block with 3 lines.
|
||||||
|
page.insert_text((90, 200), "Name", fontsize=12)
|
||||||
|
page.insert_text((220, 200), "Score", fontsize=12)
|
||||||
|
page.insert_text((350, 200), "Rank", fontsize=12)
|
||||||
|
page.insert_text((90, 220), "Alice", fontsize=12)
|
||||||
|
page.insert_text((220, 220), "95", fontsize=12)
|
||||||
|
page.insert_text((350, 220), "1", fontsize=12)
|
||||||
|
|
||||||
|
in_path = tmp_path / "input_table.pdf"
|
||||||
|
out_path = tmp_path / "output_table.pdf"
|
||||||
|
doc.save(str(in_path))
|
||||||
|
doc.close()
|
||||||
|
|
||||||
|
from translators.pdf_translator import PDFTranslator
|
||||||
|
translator = PDFTranslator()
|
||||||
|
|
||||||
|
def _flexible_translate(*args, **kwargs):
|
||||||
|
if args and hasattr(args[0], "__iter__") and not isinstance(args[0], str):
|
||||||
|
first = args[0][0] if len(args[0]) else None
|
||||||
|
if first is not None and hasattr(first, "text"):
|
||||||
|
class _R:
|
||||||
|
def __init__(self, t): self.translated_text = t
|
||||||
|
return [_R(r.text) for r in args[0]]
|
||||||
|
return list(args[0]) if args else []
|
||||||
|
|
||||||
|
mock = MagicMock()
|
||||||
|
mock.__class__.__name__ = "MagicMock"
|
||||||
|
mock.translate_batch = MagicMock(side_effect=_flexible_translate)
|
||||||
|
translator.set_provider(mock)
|
||||||
|
translator.translate_file(
|
||||||
|
in_path, out_path,
|
||||||
|
target_language="fr",
|
||||||
|
source_language="en",
|
||||||
|
)
|
||||||
|
|
||||||
|
out_doc = fitz.open(str(out_path))
|
||||||
|
out_page = out_doc[0]
|
||||||
|
text_blocks = [
|
||||||
|
b for b in out_page.get_text("dict").get("blocks", [])
|
||||||
|
if b.get("type") == 0
|
||||||
|
]
|
||||||
|
# We should see 2 row-blocks, each containing 3 cell-lines
|
||||||
|
assert len(text_blocks) == 2, (
|
||||||
|
f"Expected 2 row-blocks, got {len(text_blocks)}"
|
||||||
|
)
|
||||||
|
for row_block in text_blocks:
|
||||||
|
lines = row_block.get("lines", [])
|
||||||
|
assert len(lines) == 3, (
|
||||||
|
f"Row block should have 3 cells, got {len(lines)}"
|
||||||
|
)
|
||||||
|
# Each cell's x0 should be one of the 3 column positions
|
||||||
|
cell_x0s = sorted(line["bbox"][0] for line in lines)
|
||||||
|
assert cell_x0s == [90.0, 220.0, 350.0], (
|
||||||
|
f"Cell x0s are {cell_x0s}, expected [90, 220, 350]"
|
||||||
|
)
|
||||||
|
out_doc.close()
|
||||||
|
|||||||
@@ -454,12 +454,28 @@ class PDFTranslator:
|
|||||||
return output_path
|
return output_path
|
||||||
|
|
||||||
def _extract_text_blocks(self, page) -> List[Dict]:
|
def _extract_text_blocks(self, page) -> List[Dict]:
|
||||||
"""Extract text blocks with position, font, and color information."""
|
"""Extract text blocks with position, font, and color information.
|
||||||
|
|
||||||
|
Track B3.9: when a PDF "block" contains multiple LINES at the
|
||||||
|
same y (typical of table rows where each cell is a separate line
|
||||||
|
in the block), split it into one sub-block per line. This
|
||||||
|
preserves the column structure so a translated table row stays
|
||||||
|
a table row instead of collapsing into a single left-aligned
|
||||||
|
column.
|
||||||
|
|
||||||
|
The detection is: if a block has >= 2 lines whose y0 is within
|
||||||
|
``SAME_ROW_Y_TOLERANCE`` of each other, treat it as a
|
||||||
|
multi-column row and emit one sub-block per line. Blocks whose
|
||||||
|
lines are vertically stacked (multi-line paragraphs) keep the
|
||||||
|
old behavior.
|
||||||
|
"""
|
||||||
import fitz
|
import fitz
|
||||||
|
|
||||||
blocks = []
|
blocks = []
|
||||||
data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
|
data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
|
||||||
|
|
||||||
|
SAME_ROW_Y_TOLERANCE = 3.0 # points
|
||||||
|
|
||||||
for block in data.get("blocks", []):
|
for block in data.get("blocks", []):
|
||||||
if block.get("type") != 0:
|
if block.get("type") != 0:
|
||||||
continue
|
continue
|
||||||
@@ -468,6 +484,65 @@ class PDFTranslator:
|
|||||||
if not lines:
|
if not lines:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# B3.9: detect table-row pattern. A row is multiple lines
|
||||||
|
# at the same y (different x). Multi-line paragraphs have
|
||||||
|
# lines at different y (same x).
|
||||||
|
line_y0s = [line["bbox"][1] for line in lines]
|
||||||
|
same_y = all(
|
||||||
|
abs(y - line_y0s[0]) <= SAME_ROW_Y_TOLERANCE
|
||||||
|
for y in line_y0s
|
||||||
|
)
|
||||||
|
has_horizontal_layout = (
|
||||||
|
same_y and len(lines) > 1
|
||||||
|
and any(
|
||||||
|
abs(lines[i]["bbox"][0] - lines[0]["bbox"][0]) > 5
|
||||||
|
for i in range(1, len(lines))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if has_horizontal_layout:
|
||||||
|
# Multi-column row: emit one block per line.
|
||||||
|
for line in lines:
|
||||||
|
span_parts = []
|
||||||
|
spans_info = []
|
||||||
|
for span in line.get("spans", []):
|
||||||
|
text = span.get("text", "")
|
||||||
|
if text:
|
||||||
|
span_parts.append(text)
|
||||||
|
spans_info.append({
|
||||||
|
"size": span.get("size", 12),
|
||||||
|
"font": span.get("font", "Helvetica"),
|
||||||
|
"color": span.get("color", 0),
|
||||||
|
"flags": span.get("flags", 0),
|
||||||
|
"origin": span.get("origin", (0, 0)),
|
||||||
|
})
|
||||||
|
if not span_parts:
|
||||||
|
continue
|
||||||
|
line_text = "".join(span_parts).strip()
|
||||||
|
if not line_text:
|
||||||
|
continue
|
||||||
|
avg_size = (
|
||||||
|
sum(s["size"] for s in spans_info) / len(spans_info)
|
||||||
|
if spans_info else 12.0
|
||||||
|
)
|
||||||
|
first_color = spans_info[0]["color"] if spans_info else 0
|
||||||
|
is_bold = any(s["flags"] & 16 for s in spans_info)
|
||||||
|
is_italic = any(s["flags"] & 2 for s in spans_info)
|
||||||
|
blocks.append({
|
||||||
|
"bbox": tuple(line["bbox"]),
|
||||||
|
"text": line_text,
|
||||||
|
"font_size": round(avg_size, 1),
|
||||||
|
"color": first_color,
|
||||||
|
"is_bold": is_bold,
|
||||||
|
"is_italic": is_italic,
|
||||||
|
"line_count": 1,
|
||||||
|
"translated": None,
|
||||||
|
"sub_bboxes": [tuple(line["bbox"])],
|
||||||
|
"_is_table_cell": True,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Standard multi-line paragraph: join lines with \n.
|
||||||
line_parts = []
|
line_parts = []
|
||||||
spans_info = []
|
spans_info = []
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user