diff --git a/sample_files/test_corpus/test_pdf_translated.pdf b/sample_files/test_corpus/test_pdf_translated.pdf index 6a525fc..649f42b 100644 Binary files a/sample_files/test_corpus/test_pdf_translated.pdf and b/sample_files/test_corpus/test_pdf_translated.pdf differ diff --git a/tests/test_translators/test_b3_6_pdf_drawings.py b/tests/test_translators/test_b3_6_pdf_drawings.py index d2ba681..017fd84 100644 --- a/tests/test_translators/test_b3_6_pdf_drawings.py +++ b/tests/test_translators/test_b3_6_pdf_drawings.py @@ -574,3 +574,179 @@ class TestB3_8MaxExpandYNegativeClamp: f"expected near 70" ) out_doc.close() + + +# ============================================================================ +# Track B3.9 — table row preservation (column structure survives translation) +# ============================================================================ + +class TestB3_9TableRowSplitting: + """Track B3.9: a PDF 'block' that contains multiple LINES at the + SAME y but different x positions is a table row. The extractor must + split it into one sub-block per line so the column structure + survives translation. + + Without B3.9, the whole row is treated as one paragraph and + `insert_textbox` writes the text left-aligned at x0, collapsing + all columns into one.""" + + def test_horizontal_layout_detected(self, pdf_mod): + """A block with lines at the same y and different x is + detected as a horizontal layout (table row).""" + import fitz + translator = pdf_mod.PDFTranslator() + # Mock page that returns one block with 3 lines at same y + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (90, 188, 460, 203), + "lines": [ + {"bbox": (90, 188, 162, 203), "spans": [ + {"text": "Col 1", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (90, 188)}, + ]}, + {"bbox": (220, 188, 292, 203), "spans": [ + {"text": "Col 2", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (220, 188)}, + ]}, + {"bbox": (350, 188, 460, 203), "spans": [ + {"text": "Col 3", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (350, 188)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + # Should be split into 3 separate blocks (one per column) + assert len(blocks) == 3, f"Expected 3 blocks, got {len(blocks)}" + assert blocks[0]["text"] == "Col 1" + assert blocks[1]["text"] == "Col 2" + assert blocks[2]["text"] == "Col 3" + # Each block should have its own bbox (not the merged row bbox) + assert blocks[0]["bbox"][0] == 90 + assert blocks[1]["bbox"][0] == 220 + assert blocks[2]["bbox"][0] == 350 + + def test_vertical_layout_kept_as_one_block(self, pdf_mod): + """A block with lines at DIFFERENT y (same x) is a multi-line + paragraph. It must NOT be split.""" + import fitz + translator = pdf_mod.PDFTranslator() + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (72, 100, 540, 150), + "lines": [ + {"bbox": (72, 100, 540, 115), "spans": [ + {"text": "Line 1", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 100)}, + ]}, + {"bbox": (72, 120, 540, 135), "spans": [ + {"text": "Line 2", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 120)}, + ]}, + {"bbox": (72, 140, 540, 150), "spans": [ + {"text": "Line 3", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 140)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + # Should remain 1 block with the 3 lines joined + assert len(blocks) == 1 + assert blocks[0]["text"] == "Line 1\nLine 2\nLine 3" + assert blocks[0]["line_count"] == 3 + + def test_single_line_block_unchanged(self, pdf_mod): + """A block with a single line is unchanged (no split).""" + import fitz + translator = pdf_mod.PDFTranslator() + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (72, 100, 300, 115), + "lines": [ + {"bbox": (72, 100, 300, 115), "spans": [ + {"text": "Just one line", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 100)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + assert len(blocks) == 1 + assert blocks[0]["text"] == "Just one line" + + def test_table_cell_each_at_own_x(self, tmp_path): + """End-to-end: a 3-column table in a PDF must be translated + with each cell staying at its original x position. + + Note: PyMuPDF re-groups the cells into row-blocks when reading + the output back (3 lines per block), but each LINE (cell) must + be at its original x position. The test checks the line x0 + values, not the block count. + """ + import fitz + doc = fitz.open() + page = doc.new_page(width=612, height=792) + # Build a 3-col table row by inserting text at 3 x positions on + # the same y. PyMuPDF groups them as a single block with 3 lines. + page.insert_text((90, 200), "Name", fontsize=12) + page.insert_text((220, 200), "Score", fontsize=12) + page.insert_text((350, 200), "Rank", fontsize=12) + page.insert_text((90, 220), "Alice", fontsize=12) + page.insert_text((220, 220), "95", fontsize=12) + page.insert_text((350, 220), "1", fontsize=12) + + in_path = tmp_path / "input_table.pdf" + out_path = tmp_path / "output_table.pdf" + doc.save(str(in_path)) + doc.close() + + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + + def _flexible_translate(*args, **kwargs): + if args and hasattr(args[0], "__iter__") and not isinstance(args[0], str): + first = args[0][0] if len(args[0]) else None + if first is not None and hasattr(first, "text"): + class _R: + def __init__(self, t): self.translated_text = t + return [_R(r.text) for r in args[0]] + return list(args[0]) if args else [] + + mock = MagicMock() + mock.__class__.__name__ = "MagicMock" + mock.translate_batch = MagicMock(side_effect=_flexible_translate) + translator.set_provider(mock) + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + text_blocks = [ + b for b in out_page.get_text("dict").get("blocks", []) + if b.get("type") == 0 + ] + # We should see 2 row-blocks, each containing 3 cell-lines + assert len(text_blocks) == 2, ( + f"Expected 2 row-blocks, got {len(text_blocks)}" + ) + for row_block in text_blocks: + lines = row_block.get("lines", []) + assert len(lines) == 3, ( + f"Row block should have 3 cells, got {len(lines)}" + ) + # Each cell's x0 should be one of the 3 column positions + cell_x0s = sorted(line["bbox"][0] for line in lines) + assert cell_x0s == [90.0, 220.0, 350.0], ( + f"Cell x0s are {cell_x0s}, expected [90, 220, 350]" + ) + out_doc.close() diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 36ad4de..80a9003 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -454,12 +454,28 @@ class PDFTranslator: return output_path def _extract_text_blocks(self, page) -> List[Dict]: - """Extract text blocks with position, font, and color information.""" + """Extract text blocks with position, font, and color information. + + Track B3.9: when a PDF "block" contains multiple LINES at the + same y (typical of table rows where each cell is a separate line + in the block), split it into one sub-block per line. This + preserves the column structure so a translated table row stays + a table row instead of collapsing into a single left-aligned + column. + + The detection is: if a block has >= 2 lines whose y0 is within + ``SAME_ROW_Y_TOLERANCE`` of each other, treat it as a + multi-column row and emit one sub-block per line. Blocks whose + lines are vertically stacked (multi-line paragraphs) keep the + old behavior. + """ import fitz blocks = [] data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE) + SAME_ROW_Y_TOLERANCE = 3.0 # points + for block in data.get("blocks", []): if block.get("type") != 0: continue @@ -468,6 +484,65 @@ class PDFTranslator: if not lines: continue + # B3.9: detect table-row pattern. A row is multiple lines + # at the same y (different x). Multi-line paragraphs have + # lines at different y (same x). + line_y0s = [line["bbox"][1] for line in lines] + same_y = all( + abs(y - line_y0s[0]) <= SAME_ROW_Y_TOLERANCE + for y in line_y0s + ) + has_horizontal_layout = ( + same_y and len(lines) > 1 + and any( + abs(lines[i]["bbox"][0] - lines[0]["bbox"][0]) > 5 + for i in range(1, len(lines)) + ) + ) + + if has_horizontal_layout: + # Multi-column row: emit one block per line. + for line in lines: + span_parts = [] + spans_info = [] + for span in line.get("spans", []): + text = span.get("text", "") + if text: + span_parts.append(text) + spans_info.append({ + "size": span.get("size", 12), + "font": span.get("font", "Helvetica"), + "color": span.get("color", 0), + "flags": span.get("flags", 0), + "origin": span.get("origin", (0, 0)), + }) + if not span_parts: + continue + line_text = "".join(span_parts).strip() + if not line_text: + continue + avg_size = ( + sum(s["size"] for s in spans_info) / len(spans_info) + if spans_info else 12.0 + ) + first_color = spans_info[0]["color"] if spans_info else 0 + is_bold = any(s["flags"] & 16 for s in spans_info) + is_italic = any(s["flags"] & 2 for s in spans_info) + blocks.append({ + "bbox": tuple(line["bbox"]), + "text": line_text, + "font_size": round(avg_size, 1), + "color": first_color, + "is_bold": is_bold, + "is_italic": is_italic, + "line_count": 1, + "translated": None, + "sub_bboxes": [tuple(line["bbox"])], + "_is_table_cell": True, + }) + continue + + # Standard multi-line paragraph: join lines with \n. line_parts = [] spans_info = []