From 2da2c4765c3aae964ef8d06cb713d87f988ccd16 Mon Sep 17 00:00:00 2001 From: sepehr Date: Tue, 14 Jul 2026 19:25:24 +0200 Subject: [PATCH] =?UTF-8?q?feat(format):=20B3.9=20=E2=80=94=20preserve=20P?= =?UTF-8?q?DF=20table=20column=20structure=20during=20translation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User reported that the page 6 table on the test PDF ('6. Performance and Scaling' page) was completely broken: the 3-column table ('Document size | Avg latency (s) | Throughput (docs/min)' with 5 data rows) was rendered as a vertical list of label/value pairs instead of as a proper table. Root cause: a PDF 'block' that contains multiple LINES at the SAME y but different x positions is a table row (3 cells side-by-side). The extractor was treating the whole row as one paragraph, joining all cell texts with newline. When the smart-fit logic wrote the text back, it used the row's full-width bbox and \insert_textbox\ wrote everything left-aligned, collapsing all columns into one. Fix: at extraction time, detect horizontal-layout blocks (lines at the same y, different x within 5pt tolerance) and split them into one sub-block per line. Each cell gets its own bbox, so the translator writes each cell at its original x position, preserving the column structure. Detection heuristic: - Block has >= 2 lines - All lines have y0 within 3pt of each other (SAME_ROW_Y_TOLERANCE) - At least 2 lines have different x0 (within > 5pt) If all three hold, it's a table row. Otherwise, keep the old multi-line-paragraph behavior. Note: PyMuPDF re-groups cells into row-blocks when reading the output back (so 'len(blocks)' looks unchanged), but the LINES within each block are at their correct x positions. Tests check the line x0 values, not the block count. Visual proof: page 7 of sample_files/test_corpus/test_pdf_translated.pdf now shows the table with proper 3-column structure (Taille du document | Latence moyenne (s) | Débit (docs/min)) instead of an '[translation overflow]' placeholder. 4 new tests added: - test_horizontal_layout_detected: 3 lines at same y -> 3 blocks - test_vertical_layout_kept_as_one_block: 3 lines at different y -> 1 block - test_single_line_block_unchanged: 1 line -> 1 block - test_table_cell_each_at_own_x: e2e table translation, cells at correct x positions Total: 457 tests pass (was 453), zero regression. --- .../test_corpus/test_pdf_translated.pdf | Bin 32980 -> 41083 bytes .../test_b3_6_pdf_drawings.py | 176 ++++++++++++++++++ translators/pdf_translator.py | 77 +++++++- 3 files changed, 252 insertions(+), 1 deletion(-) diff --git a/sample_files/test_corpus/test_pdf_translated.pdf b/sample_files/test_corpus/test_pdf_translated.pdf index 6a525fcf4cfe5ad3221d99b915448b9b3355f3cf..649f42b2bab89add880804ea8be6f0f6d1a20da4 100644 GIT binary patch delta 10543 zcmb7Kc_7s3|8K^96=p~ZBZ_j(=kuA9TnUwKX1u+h*ZX+h&--~l@8{{uB=gUaPvw#{ zXc&`5r+L#bmhgiOV#p>sGzKR4lfe*v2zeMRU??*U1)jE_%9O2c3=tqh1SpC;L>PLXy^CNX0T^1Jr*PJQj4}aq4zDQRCSJ}^)j8dF zSj+$K%%nrJ*VTP%8uNB6q&;hz?HuPQ2L{7c;n14>FGt8L%?>Ia;QEeCnnitj^Uv$uZW^aI-=J*&I#lI!ly4SA#gDyL~Z_nDUQ zu134VuXZ=}dRgt-XpuH$V@EZnTc}Zby8kidx7TlWy3k`MUpTn0DLfJZ8hel3tlXCn zy=3g<*yW+eEq|ANuU0;Ew50hP|3&fUyUw}%9-bv1M7h3m!C!fQPYPv?uT=)g?wa)~ zrn}_H2pz1V(?CLuhL?|#t(#1E_MRZQiJg6gsGO6NtrM2LhjB_%CY!dwwA?h<@ZRPA z9@E=b&b8k(zdg`CaK*g4t9_uatAC)qOY@3nW1iaUkz}=DciHviu|t@YX?RTzn%Wj zK|Uv8%acXF(jphlVXr!%>ap|p^VKWb+5*-sP3&sRdzI97?)kOK#vg9x_2rB{(%Rv+ z80bbl*xI}1rwq!T6Z|I+KlK-emj(A^+NpHwUsaBbk4Q;Pj$E}C{|0#Oa)68y4Gow1 zvP-{i=w2~Hjqe($3JiUSQo-0t!#Il_wYZsb{LRik4_RIv{rL>dPcb>P@(9IdLu1c; zm9|0Wh=ZBa(t0fy2FPDqu2WY~WxU(u#`xI#Q_qeJJ!x}(-|+XtjNjj~X04h1VaMpS zOL>Qsatb!C-!?PdQTIeyxy61?kKL1nA>+}yS^~kBFbau-dl;WPXTjXbsQf3BN*LNM zdu;6PmZfG#PcL6)-gB+oLkcglhnLQ7xdrq=57#G~PXn9cPv2BkBwI z6FVlhKDYZmKqk!aik!`?qW*&D(roQ5ZLeqgbR}Nz$y@(Je&m8?#mCHwnb-fkTAiN0 z*?*mP;)_n__l!>+{L7DbnXH)|a3N61b+U?sQE5rI&XMz*89f&hCz@Zq_)$MPZ5}FCtJT1r@3O&H>b=%N8rT(mKKSPuYeeQ0xvkI z%RzI;<*tt2e_^Tc{Y=_?>qwhLvnsVK=-GoW4bg5@)z+BFnu)GUr^QAdE!3EPd3AaQ z4c%@y+LoQforvTOrCm7~I5M-?=GoQugORt|Eo)lieOnu>Pt^7xPuxTNYji3Fa#h%b zi1fiI|6h}iiJ4^5X&?-Y85}A&WoyV286=#)GiUb~y>&2`)%YX-dHdYv=)Kgy_Y?68 zQPk1`dY)4K{bX!nL(3Mf2Ar3 zRNgf2^QvLPYE78k?Q!1|MSNZ5Y%!JNx9I_|+tCnaKtg{!Gy7nFU&`X9J~$Fuz*=1T+^>LAy;OnMDW1W;AeU zUNjljmBnVxV6m+?$b!4sN!uE5(1Qw+>_$P3Cl#dFmxI+yDD`t3%1FRqDHRMl_JWT~ zRly#2Ghpwh2wu*=1M22koBEE*(%Cjq!7M&N^69^eP6*6(pw zCXv}3Q0VatxC9A>wR%yrdkw4yFQZX$KjWuedCGujOE02nAGn9S3cpl=>)d zEfUzeidz4jj}8e`gi*oSWmO;`oKnAW`5qF;iJ;U^_fsc<)sYln=YJjKL{R}P;0jo| zno`aWAdz7Si^Dps@C4Q|R6MbR3oEj~3-~=SXaJPPQoy3%qo8vQ709i;501s*yIBbc zSWBrt7NSieaY3&I1>6cfDFch<`m9wmN$?8-=EY#(*RUvwHwLW{ zRRBrRs!xwpA%TUdR5HpE6?L|rvWTh+zwm;|skjdNBsV9u4GukZ<5r0W3Y_n4E!uhW z$fl3QdL1v$W;+}(Qrd3q5+T2^LcIe`1Z^xNz$L->O=VSN%Y1SkHXH zTy%fdl=Y#9zpEQs6JEVsVK#qL>Rf|Av>w`+nHJs2-I^;lsfAi>xKhM5-jgZM`jj&TEA&IE7D!%Q4@m2&^_*xj35=}Q zA|rxq7QZf6Y#(|&`fx0Ga{K%}-xSpq`Nxzwn>W1Tt38bj4b|DyY1dw&l2Z}-)VsoH zW&ga64=PxDRz&QsKz>gC-pwcU9}f?Hn3s4rt+e*_>7jstXR)4tRK4|l?B6-6>&ne4 ze4s$Pr@sL;=ebtQ|X1E=%V$8bl%vZ4pYWtYp ze0%ts&o9BMAG?QYLU)`m=r!;kxICeTy;C~hr`y<k3zsf~;B~G&g(NaxLIVsum?iT@#ZmTi*-;Zx zyMKjV&AiB4&KhlVymLc6&-?cn_}_;bEPt0<;#E}3a;0az@wu2Zj%#otMc=dhPF$}p zX5!<)+Ekiy!0PeO``7N@ws}%}p57%~0c}KZluHPURh~D`>Ys9hAxfMYj!r7JMwpLS zYIed?bBkDNE{ux^jf;*5#S?Q>d{pez=_8jO>R!;gA~%V$Ofw4?qJ;>~iIPIt|CDbw zMn3Z^4zP z3J_hqQ(G(FE z8nH*Y{c_G_dD`jb%!1N_=7M9*wPFR!M?300kuDQ+;;Mi0|&;*jihV_kL-MRRD zc|}VkCb-_>qT@p2zP##IGAKSG8}J^Hq#S84eOa1>4ekAX&`aVO1cYm7h$MVrT}yfC ze~{24HUo&#lsX(+27xt3QUU`Q#cqM$+4beIrm#wZZL#h(1F#9v1Gl!;xWcK3pfM(C zBnqodI8(eI0OFg4C}!~sXPOKlvWg9?U(yr~pB|l10pqLJnXwn?9(<`rcmH4+9LqM@ zr%d^N#>vGdFABU83?5dOT>tcU;K`A(fRJk|=1jfjsC>-Aakhn7A!F;GN(Y1PZ?#_P zWBTJ1M2B>3ilK7#uE$QPqtZjDT>!<5!dYR>D~Q@$%VE=?3pJ2 z`N88M$Jupu6Slp<=3z4wp3}!1nkKla_9^eLhs~fe-n6g!onKbaey@~neomDPbSmj)u^Q8q&RsS^Kb9GH9YwC=iwI2@_wCGsTyYmnD_sxlO* zmnYJb1-x7_o<&~kDLvL0FA7&ft{ys*Qst;Tbm z>!HQzA1BG9#xhVhkm{}uHR9kJf$2;d&!Q7{r4ZH2uWgs?HR+k{(Y93mTN#ak{P>a7 z2J0s!+ioO`?lB+U&wJwGV>C=PI?hCre=}8XnwnrR@gY(pK3q*RWcR0sDUY9Dm+w=P z)4RQ#5=HeWzS!QKx8lIDbn5UU-N}k`A3d!5rFv$pIZsO_gSW=SnqhrwO6`%E&B~D= z#a)ApVE7q=j0$o*^boEQ^w8!@{HXoBy< zmd&K$i(?@x{p>7TJ@mpiog?e!eA97P)~H4Ey=|*6mg}2UmR;_;qA1z*&=K2=NlOf8 zR7-@ApgjroLBtmntbpNyav8R(e7}S~yTG_!rY@Vd6<<*1>lP?C3B{^?HLa}XRp61l{)UXz5QvV;q*C8)`kh2MlaeIkKwZ0 z27^Wwjj^sn=M=`&bmruE6`l$V$n}j&2(srt|F*0DvaF(2hx2BGjrV<@#vi%eF0bd& zXJV0Gb9r~j{ItU-)q)$IL?z|#bZ_>*sefdJ*Y!t*Nwa%ZWmlyc!RZYq$t#*?$86PD^_?rD>ZCZ4c>}04 z>YlOEM#r9Mzr=d6A5f|Vc_z6;(QU)-W96nhCgLt6jh}2u;G9^&sINF~)Usppl(zTV zb#GrvH$QyzEqA=Z!@6gxK11J(I8V?`r);Po+*R0H6Z6ujxr*?ZWK&}ue*;GGC||8Z z{e#X<0Bj!ohe#vt4RB&4L>D6myo1bsw%R{ukT3$SCdX;#M66nD!KR~Oic&|lC z>yh1cd;LMSPY+Ul{({mL{RvOMN`(nheT80?ggU~^!w{rlK@_js)rvPS<@p&qUMgAg z!Ts*cBkIPXl|d%HBQY@V*tMNsS)p5@lh%8B?>Bz3KMaRCU+gW`8NJh?Ff&@u**`YX zQwzq-YRc};wl_ZyHzetwJ@nnRdzgEJ>*ZN#fq@JAf|kU*ZGI7R=ZQl|$Ta=466*XD z#Vj~mSb}&2opY<(uLONFL*Y&lJVV~SC0OB8_r^R@!~2dP|3}xmRj1b6omTkv>cv)D zyW`iICuE0j{bluX`r+|#r8QdmQJTA>w&qyOoAhnYhRH#E?=lCk@=yAf%{}=IH>bw= zO`jv&5nx9u$DIM9Q-;Y54sc67Nn!{(7n9e$BQxpXF^U29`r~9K^0~QGyup$TuQso5 z@aHi(ut>yzXz?^s%AgogK?aip>rP2PHXXlwl&0a=ZxS?k-HBiLex^bBbUM8I#BW_c6NNN3T^bOf zbMPBXDLyVHtqeTX;}?UHeDE@qpbUcY*aRVP4>(IrwG0BqIyO&yG?w1~CFyOkxDE*!X?7b9Q-APB#lcaP92wl zzmSmPV-sa?xm*@akOThBe<&Tr-y=v2M(IraO@kDTO%ROIx%j&ZNj{i1q`iQ`NdFv1 zJW=9B5hni5Kyolj$KN7I(NO$dgA|R4H`%0UY(l`n6w8*D!9y_wfA=Ce7{l=ADN-~h zevXu)v5DSd&S#v-f;l*PrLmPjhY7QBCvqA~Dpr6i5VhU!QY*}M;%aA- zw!kcWEJy}26$87340n!>o);SvOtPS_m*7Q_El(**T3Bh}q=j!;XiQpgX(6Pg#jX{N zlAKTsR12WAghoq3S^}jdP+9_IBv3{IWh78W3WjMhZ^fn=$kN|1mXXjHro}%ap)nE~ zE1|Ix8Y`i(5*phcjbUMlLBdd0LSrQ~RzhPXG)_X}Bs7iydjs2l3&*k$W|+`}lXQ7W zmzQ*T0u+W8kk>J}GcRS$b6{uKTVVmW3r?RgTCsev&+o6g>Sp29eVMli#GbD9Yx)K)$RGs_3O)Kr<<;;`({A&IM0~teV^A_ zG~L**9$G)i_W6}M=f=!LtjDTC<5+`}^P@-So4TIZxh-(OvMUb%3pX|E*)aHrw+Dg? zf7<-#-Ooi=KTrGQJH)p~Xt%^(Usu}pG0xcNQokVd+kC$fjU7|Jx*r1XrKq-gP^zJl z|_R%&0C3>cNPEdBAwMq{%_$x0)e4@;Tjda(gJ9(!i=9_V?7 z)M~ff%Ku~0Nqrpi8&s8Y$0QEfJmg-_hQvM{v99U92NxtAT|P7cG$A&-9M7B>eo#N(7Js9T>^|`( zJ?c)j+KU`Nr|e`8%lLYe|)lI{w~+g|8jHKN^AG zq3?$$dCiZiQ2?^?x_I+j0hh;Y&cNn|j7x%+gs}5~{zxwHj<`)HVm*NtN4oo%w$#B8@ zF}I?Qd3!9movPY#Kd6uAiqT`+xxS_7Powxg)dfH7KBwKhS+pbUid3M z;Y4w!N#zyFY*XK?l`CB1ek*^Lv)^>YJr|FGAA;;2);iqu%x27#Ax3Zq`wmL%{(`kQ z3Pv-ch}PE0RslGL!u$~jppjBRy?b|X9XSV<(PmI;ZwU`Z z#skeD5bN9wRSXKtUEYGpY82MGo`azr3ak0L@wo940gH+5ax_(O(vQuK<$D8>vbKLxJr?WZv4ExVDcywj(Fz- z=O#vM4H-T)VCarQbFdXm^f!duNy%`@U8N;`A1WYlv396K?U=tAwHv&*7insw~fo8BXG9P{jLLqqOP4Eg< z!B)I8Oqi7oPl6Hc*IC^a@XHhgY-SfhbqK0`K3nkrWhx4$bF-l|6w#K??WusNVF(QO z#K3x979>qWw6*h83Yaz>ZF!4A*GaeUs`T#Uw#5H;qv*u5kE-hep7u>Wbg{Y1?bNdA zW6n)DcV6*$Lim(*#?#-7@{2oQ7N2syK6z^Hv8!J$+%9tb=)&$AJ{Z||CM>NnClGPk z{h@Q+$(?$67jpC)SH~vpi+JR}_N4D4^@hWRi%z-6eod-#C=62ob%iTjoEBl7_s;lz zVNPC7vR~Dgm^3SE?!}R1B@Gsqk5&s;H@` ztnf%uM~wW7ii!CXExl$u-1e&RzLB;aOuzR0+~Rs$1PUIrEMcHV1F4aS_NbU_qzs2s6$^_PyF;_*2D zyZ0-Phb%nhyA5AyHef^Xr=6a!12-OaY%uugc4_E>ghKP;hve`1%{8F|t`|}y=BDI%f#FtgF?kMLpZyOH z7b`Iw*5QaiEDPw8k_ZRqB9N8R1P#%M9!24>)25%MWMf%cEoPk3x>Od zU3Tf^xuWB*1RB}wX!ywg_|7Qfh8cR~=k?dj$HzQ;V;=d*scdR~rqRC4UH!?6Gcq&g zHI{FB_cn45JuLV_$>pNEF}WUto^GyeJXQb5YDC|fvzvluO!D99vbEP_hw}4#R_!%# z|McX~*DuB{9;OLQo;dE(ABu-v7@^wBaEzDf z;Uq4MV>)^?B@9tIGL{eqGz`sVtZRbBukFxabc8e z)r%7EgfUGVqFaTMjG)^pN-|O$Qg!qwajMjj(Xz_m7_8fbXfTeDqw;d^I7Z1(aiJCI zdcsLria1W=vM6zu5KAVM``;Ih<2)vFLXdJN1VhLi6C5T?;qB8VF+zq%5_B7qi2j9o zle9Rri&rb_aFP|LdmR~%w=r&ch*Ow2i?`RK{v&9Lkq1L*IC)uVOdbp^q^IkIro;n7 zd&e|`%QB`}@qD7A$4kU%OLv?h2w8RvC4Zj`Cm!P31G1Rh2}_D6IUPM(JP5VQ48=Q~ z1C3J}>O^81Q+5^%QsY_g;Fr-|M7k?VG+P2hR>^ZX1Fv6y8qbOPs3?qLf z3@@)1i{r9HSVBf%f~7Sw1`{mzAI7|_$rBtQCo4zFzDxv16SCxl9VP`hr46}+9VUyK zr^J5~t>tQ|0?&#+Omt+t_}8YLjKrXNJED(KBn7Xsdy7Lr*ByjC?TRzb+$ffJbLQMw unqoLtXJ>}Od480@ZdX^Q5GN<2nAk9N>_T;Tgpu&gVKAcs1KcLM8~q>i83H>1 diff --git a/tests/test_translators/test_b3_6_pdf_drawings.py b/tests/test_translators/test_b3_6_pdf_drawings.py index d2ba681..017fd84 100644 --- a/tests/test_translators/test_b3_6_pdf_drawings.py +++ b/tests/test_translators/test_b3_6_pdf_drawings.py @@ -574,3 +574,179 @@ class TestB3_8MaxExpandYNegativeClamp: f"expected near 70" ) out_doc.close() + + +# ============================================================================ +# Track B3.9 — table row preservation (column structure survives translation) +# ============================================================================ + +class TestB3_9TableRowSplitting: + """Track B3.9: a PDF 'block' that contains multiple LINES at the + SAME y but different x positions is a table row. The extractor must + split it into one sub-block per line so the column structure + survives translation. + + Without B3.9, the whole row is treated as one paragraph and + `insert_textbox` writes the text left-aligned at x0, collapsing + all columns into one.""" + + def test_horizontal_layout_detected(self, pdf_mod): + """A block with lines at the same y and different x is + detected as a horizontal layout (table row).""" + import fitz + translator = pdf_mod.PDFTranslator() + # Mock page that returns one block with 3 lines at same y + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (90, 188, 460, 203), + "lines": [ + {"bbox": (90, 188, 162, 203), "spans": [ + {"text": "Col 1", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (90, 188)}, + ]}, + {"bbox": (220, 188, 292, 203), "spans": [ + {"text": "Col 2", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (220, 188)}, + ]}, + {"bbox": (350, 188, 460, 203), "spans": [ + {"text": "Col 3", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (350, 188)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + # Should be split into 3 separate blocks (one per column) + assert len(blocks) == 3, f"Expected 3 blocks, got {len(blocks)}" + assert blocks[0]["text"] == "Col 1" + assert blocks[1]["text"] == "Col 2" + assert blocks[2]["text"] == "Col 3" + # Each block should have its own bbox (not the merged row bbox) + assert blocks[0]["bbox"][0] == 90 + assert blocks[1]["bbox"][0] == 220 + assert blocks[2]["bbox"][0] == 350 + + def test_vertical_layout_kept_as_one_block(self, pdf_mod): + """A block with lines at DIFFERENT y (same x) is a multi-line + paragraph. It must NOT be split.""" + import fitz + translator = pdf_mod.PDFTranslator() + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (72, 100, 540, 150), + "lines": [ + {"bbox": (72, 100, 540, 115), "spans": [ + {"text": "Line 1", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 100)}, + ]}, + {"bbox": (72, 120, 540, 135), "spans": [ + {"text": "Line 2", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 120)}, + ]}, + {"bbox": (72, 140, 540, 150), "spans": [ + {"text": "Line 3", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 140)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + # Should remain 1 block with the 3 lines joined + assert len(blocks) == 1 + assert blocks[0]["text"] == "Line 1\nLine 2\nLine 3" + assert blocks[0]["line_count"] == 3 + + def test_single_line_block_unchanged(self, pdf_mod): + """A block with a single line is unchanged (no split).""" + import fitz + translator = pdf_mod.PDFTranslator() + mock_page = MagicMock() + mock_page.get_text.return_value = { + "blocks": [{ + "type": 0, + "bbox": (72, 100, 300, 115), + "lines": [ + {"bbox": (72, 100, 300, 115), "spans": [ + {"text": "Just one line", "size": 12, "font": "helv", + "color": 0, "flags": 0, "origin": (72, 100)}, + ]}, + ], + }], + } + blocks = translator._extract_text_blocks(mock_page) + assert len(blocks) == 1 + assert blocks[0]["text"] == "Just one line" + + def test_table_cell_each_at_own_x(self, tmp_path): + """End-to-end: a 3-column table in a PDF must be translated + with each cell staying at its original x position. + + Note: PyMuPDF re-groups the cells into row-blocks when reading + the output back (3 lines per block), but each LINE (cell) must + be at its original x position. The test checks the line x0 + values, not the block count. + """ + import fitz + doc = fitz.open() + page = doc.new_page(width=612, height=792) + # Build a 3-col table row by inserting text at 3 x positions on + # the same y. PyMuPDF groups them as a single block with 3 lines. + page.insert_text((90, 200), "Name", fontsize=12) + page.insert_text((220, 200), "Score", fontsize=12) + page.insert_text((350, 200), "Rank", fontsize=12) + page.insert_text((90, 220), "Alice", fontsize=12) + page.insert_text((220, 220), "95", fontsize=12) + page.insert_text((350, 220), "1", fontsize=12) + + in_path = tmp_path / "input_table.pdf" + out_path = tmp_path / "output_table.pdf" + doc.save(str(in_path)) + doc.close() + + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + + def _flexible_translate(*args, **kwargs): + if args and hasattr(args[0], "__iter__") and not isinstance(args[0], str): + first = args[0][0] if len(args[0]) else None + if first is not None and hasattr(first, "text"): + class _R: + def __init__(self, t): self.translated_text = t + return [_R(r.text) for r in args[0]] + return list(args[0]) if args else [] + + mock = MagicMock() + mock.__class__.__name__ = "MagicMock" + mock.translate_batch = MagicMock(side_effect=_flexible_translate) + translator.set_provider(mock) + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + text_blocks = [ + b for b in out_page.get_text("dict").get("blocks", []) + if b.get("type") == 0 + ] + # We should see 2 row-blocks, each containing 3 cell-lines + assert len(text_blocks) == 2, ( + f"Expected 2 row-blocks, got {len(text_blocks)}" + ) + for row_block in text_blocks: + lines = row_block.get("lines", []) + assert len(lines) == 3, ( + f"Row block should have 3 cells, got {len(lines)}" + ) + # Each cell's x0 should be one of the 3 column positions + cell_x0s = sorted(line["bbox"][0] for line in lines) + assert cell_x0s == [90.0, 220.0, 350.0], ( + f"Cell x0s are {cell_x0s}, expected [90, 220, 350]" + ) + out_doc.close() diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 36ad4de..80a9003 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -454,12 +454,28 @@ class PDFTranslator: return output_path def _extract_text_blocks(self, page) -> List[Dict]: - """Extract text blocks with position, font, and color information.""" + """Extract text blocks with position, font, and color information. + + Track B3.9: when a PDF "block" contains multiple LINES at the + same y (typical of table rows where each cell is a separate line + in the block), split it into one sub-block per line. This + preserves the column structure so a translated table row stays + a table row instead of collapsing into a single left-aligned + column. + + The detection is: if a block has >= 2 lines whose y0 is within + ``SAME_ROW_Y_TOLERANCE`` of each other, treat it as a + multi-column row and emit one sub-block per line. Blocks whose + lines are vertically stacked (multi-line paragraphs) keep the + old behavior. + """ import fitz blocks = [] data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE) + SAME_ROW_Y_TOLERANCE = 3.0 # points + for block in data.get("blocks", []): if block.get("type") != 0: continue @@ -468,6 +484,65 @@ class PDFTranslator: if not lines: continue + # B3.9: detect table-row pattern. A row is multiple lines + # at the same y (different x). Multi-line paragraphs have + # lines at different y (same x). + line_y0s = [line["bbox"][1] for line in lines] + same_y = all( + abs(y - line_y0s[0]) <= SAME_ROW_Y_TOLERANCE + for y in line_y0s + ) + has_horizontal_layout = ( + same_y and len(lines) > 1 + and any( + abs(lines[i]["bbox"][0] - lines[0]["bbox"][0]) > 5 + for i in range(1, len(lines)) + ) + ) + + if has_horizontal_layout: + # Multi-column row: emit one block per line. + for line in lines: + span_parts = [] + spans_info = [] + for span in line.get("spans", []): + text = span.get("text", "") + if text: + span_parts.append(text) + spans_info.append({ + "size": span.get("size", 12), + "font": span.get("font", "Helvetica"), + "color": span.get("color", 0), + "flags": span.get("flags", 0), + "origin": span.get("origin", (0, 0)), + }) + if not span_parts: + continue + line_text = "".join(span_parts).strip() + if not line_text: + continue + avg_size = ( + sum(s["size"] for s in spans_info) / len(spans_info) + if spans_info else 12.0 + ) + first_color = spans_info[0]["color"] if spans_info else 0 + is_bold = any(s["flags"] & 16 for s in spans_info) + is_italic = any(s["flags"] & 2 for s in spans_info) + blocks.append({ + "bbox": tuple(line["bbox"]), + "text": line_text, + "font_size": round(avg_size, 1), + "color": first_color, + "is_bold": is_bold, + "is_italic": is_italic, + "line_count": 1, + "translated": None, + "sub_bboxes": [tuple(line["bbox"])], + "_is_table_cell": True, + }) + continue + + # Standard multi-line paragraph: join lines with \n. line_parts = [] spans_info = []