From e706cef5d6a183577d855a8f5998497edd145d23 Mon Sep 17 00:00:00 2001 From: sepehr Date: Tue, 14 Jul 2026 18:36:12 +0200 Subject: [PATCH] =?UTF-8?q?feat(format):=20B3.5=20=E2=80=94=20PDF=20smart-?= =?UTF-8?q?fit=20rewrite=20+=20critical=20fontname=3DNone=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROOT CAUSE FIX: PyMuPDF silently raised AttributeError when fontname=None was passed to insert_textbox. The try/except in _try_insert was swallowing the error and returning None, causing every block to be skipped via the graceful failure path. Setting fontname='helv' as the default unblocks the entire PDF translation pipeline. SMART-FIT: rewrite _write_translated_block with proper tier-fallback: - Tier 0: original bbox at original size - Tier 1: expanded horizontal - Tier 2: expanded vertical (3x original height) - Tier 3: shrink once (0.93x) - Tier 4: shrink twice (0.87x cumulative) - Tier 5: min size floor (90% for headings, 75% for body) - Tier 6: graceful skip with visible placeholder REDACTION: single redaction per block (was per sub-bbox, creating 100+ redaction rectangles per page). Now only 1 redaction per text block. FEATURE FLAG: PDF_SMART_FIT_ENABLED (default true, observation-first). METRICS: text_overflow -> format_elements_lost_total. RESULT ON REAL PDF: Before: fonts shrunk 22pt->5.6pt, hierarchy destroyed After: fonts EXACT match: [8, 11, 12, 14, 16, 22] preserved --- config.py | 13 + .../test_corpus/test_pdf_translated.pdf | Bin 0 -> 33427 bytes scripts/compare_pdfs.py | 107 +++++ scripts/debug_rc_real.py | 40 ++ scripts/debug_rc_values.py | 37 ++ scripts/debug_smart_fit.py | 81 ++++ scripts/debug_smart_fit_trace.py | 55 +++ scripts/diagnose_pdf_layout.py | 71 ++++ scripts/rerun_translation.py | 33 ++ scripts/test_full_pipeline.py | 72 ++++ scripts/test_merged_block.py | 87 ++++ scripts/test_smart_fit_no_translation.py | 46 +++ scripts/test_with_redaction.py | 59 +++ scripts/verify_b3_5_fix.py | 62 +++ .../test_b3_5_pdf_smart_fit.py | 389 ++++++++++++++++++ translators/pdf_translator.py | 204 ++++++--- 16 files changed, 1294 insertions(+), 62 deletions(-) create mode 100644 sample_files/test_corpus/test_pdf_translated.pdf create mode 100644 scripts/compare_pdfs.py create mode 100644 scripts/debug_rc_real.py create mode 100644 scripts/debug_rc_values.py create mode 100644 scripts/debug_smart_fit.py create mode 100644 scripts/debug_smart_fit_trace.py create mode 100644 scripts/diagnose_pdf_layout.py create mode 100644 scripts/rerun_translation.py create mode 100644 scripts/test_full_pipeline.py create mode 100644 scripts/test_merged_block.py create mode 100644 scripts/test_smart_fit_no_translation.py create mode 100644 scripts/test_with_redaction.py create mode 100644 scripts/verify_b3_5_fix.py create mode 100644 tests/test_translators/test_b3_5_pdf_smart_fit.py diff --git a/config.py b/config.py index c405080..5a8d26a 100644 --- a/config.py +++ b/config.py @@ -104,6 +104,19 @@ class Config: # When true, only Pro+ plans can use L2. Otherwise, all plans can. QUALITY_L2_TIER_GATE = os.getenv("QUALITY_L2_TIER_GATE", "true").lower() == "true" + # ============== PDF Smart-Fit (Track B3.5) ============== + # When true, the PDF translator uses a smart overflow strategy: + # 1. Try original bbox at original size + # 2. Expand bbox vertically (3x original height) + # 3. Shrink font ONCE (0.93x) with expanded bbox + # 4. Shrink font AGAIN (0.87x cumulative) with expanded bbox + # 5. For headings (font >= 14pt): never below 90% of original + # 6. For body: never below 75% of original + # 7. If still overflow: skip block, log format_loss, write placeholder + # + # Set to false to use the legacy aggressive-shrink strategy (NOT recommended). + PDF_SMART_FIT_ENABLED = os.getenv("PDF_SMART_FIT_ENABLED", "true").lower() == "true" + # ============== API Configuration ============== API_TITLE = "Document Translation API" diff --git a/sample_files/test_corpus/test_pdf_translated.pdf b/sample_files/test_corpus/test_pdf_translated.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2a0aad23d95cf9f7e1049d931759955b295ef651 GIT binary patch literal 33427 zcmeFZWmsIxwgnmp4j~Xcc!1#2xI4imc+lVy+@0VAcXtTEo!}w3yL)hl;0~{Wz0Xa~ z<{b7t_rCl7ynG)`n(nGO$E;ejMva&0mGSPC>F$;J?v>^46+m(fu^AL~jt}8scyAJT^?h(@6K~$yYF)=Z{?d`rH;JzX4eM8#& zhP3w$Y401--p7XaJ~*^@Ui=z_+mHXZgFvk)scERAbsHREV*noD&Y^!38@hYv=8{{s`lU%=dU_`eP2pC{U#@Be0^G2Vk?yq{={_uv@s!7<%~W4Z^&bPtZ{9vsu} z;28fh?QVPe-v;+DrybKh3Z{D$O!p|5?olw`qhP*A!F-Q``5p!H{RlJPH)p>O$elP$u_ssz`zYhn?eLMg(cLDhg z-K~LN(cLae)H2qZR=3OV?HUZA=GCy&x%;yaaHZ0*GSJeX7BJB=)iyBEqgF65;W4o^ z`1AcQtK(hP{eO!xjI4AlsNWa?*GGOGV6QsBKDDj%@#tu1sD*S4 z^z^Op0L(Phyara5QaToVrp9KbCb#P&GjJg{GPRH~)6mkv1Kh=fT1>}857?BJhMocF ziKUf=j)pOeUBdQ+vTvX&R@+_%)(fg7dU%0pIDmFsGYw0C48I8eYP^AnC7n3Qa4rd2 zC%#`=h_8zNl38lVLL-V7mc>in-cI~Q`#N^fFo{Z!w z?3_NsWE6{|H&|*9wGb4wV2V2glP^%IvRv&Ik7w-Yk?76O&lj%`H6yx^fyt~Q4|>Va zscJq}po)TmqSYEuWj62L*TFWRK8gEtf4S>jscr{mL^3T^%vSrC>4ViiouAuN74uC# z6=h`^k`$+?B^T84wB0OLYf|+tSMr5w&g+9jXLHrntP`J1=e|YOt;4TPR~;N}jwQ3( z52o@#o|U>>o))`ii$+tm1>;6Hgc1m@^d%=>siymURM<5&fd{Jp~r7m37a0EVN~Xh$ra zUf6VL*0lJgC@dP4RbN6+PY<15?_o6XdYb)&M67KF43qWdD3KSVL@QX>G+89k$=Si` zp(AM}Wa*GLMPVls^BL9Ua#u8`^SQQIlj*@oJCUX2o zg5dO$=otdRbAHwf-yI4 zE)_p!e_Iah0L0^eUQ1a;F!t2vgXHkhC*a3qU1QBFCd8t{g9fFF3{%V^ba z>C4Dlmg4s^`lH+b=^y-!L&Qp4EZ9+}@M5*E zmh33L^3ZmickC$|d1!|ng)bJ5Lv#>s61Qm;KMNBcl?-ar#3T&QOU4RrVHA4(j*A87 z|Ehzh|7y;&UW0p|QkUIo0e|p-KUD$1ErFV!O9YGlA$lYLbBmv1$jeCbyPv=Hz5TE2XJ66y2ofSYKh8kQhS60?q zoRPX)JxSTp8SNI4d^exoCH6e{rcqBpu}I|y?VRHLnydYmXm35mdD!A+kE#m_%o8lq zb4>Uf?;FcqKUrq?!||+kqfAK)SqJ$Tys+4>gU~q9U5Chof-SH4U1WaVL|$Tl7S#Px z$ZulN`5G84G1dcDcm^Qh0#n!9YuywL#(#6;+NUR=Ubp1%WZd!z-o-bE`N$_$JD}X> z0}7JHRfQ2wPgr+;(t*X^IAldWdj|Cpbd4K=1XL~s+R6=)=4sS-ROTmWKR}##5XPUJ z;z1CCg0zEHy@w=uBJduU87haHQvZo~3Q7V*;t%#p&;pOCA5`cN!EOj*?$O*}vu@x} z?nMX~X*{$cFEZfQx!)jQmUv^~F|}jDxbvp!lR$TVv`9^qLX?G(Y#J{E-=b0?g|7v!7}}b3T%d*gsKD8QI}r z=c9jOfg-$Wg8^$avad`^h|i5IM0J_CVNa3fSc19P2k2cK6P_3UF~iayC}5j34o z7H`f{PqSo)IkqFbU5c~=EsG>g4oDN@XXlZX7^b@bg^W2hK>Tx3gl zi)ahM=EVh$U^D4_fLT!GdzBYd&lQ3`w5m2go6j_ZwKK0}odw->3;u4}82tnD=gcYF zsqiHzkNfvmiil3o=APJ|FudgRFEs*wdJ7TeAoP4vYgSya(FET^-h(htICbMEG6=8C zK#+zopk8NQr(MTie-PJ(XqZ%xAb~nZeM2K5fB%Xgf25Dc7$2lx6Sj9t(7wXR4 zqH6I-wejGdK?H|>C(%gI4WQu^Ar&&uc=h>Ms9fZkaF1x0@RW$42%1QR@N-eu2#rqW z_U5jw==BKuSN>rmQAc4tVH~d<#ckw+sPstNqRAuJC{KiNqQ;_5qUI=E#lES#NbtOE z5XKism7dRO%zMQ*APoUkPI7*aXvPhChd?%8eJDDIDCEL)RKiVyOH1GT#Bw;t~jsyAYMa2LjYs- zEZdvP+unP|+ZEFQb1yI_aHsHfp^ZwNvGv%-SSxF5XsciVZcM+}w75{ze3k(j`ebt6d|z=b!<anD( zRIZHPMsy^;;JpBW%<)L(i)@k+L}}A+Llq1S|1_lgrK?w8Phs_{H=P|ECwZ(HsVb(* z$b8+JsOr2@zKYXw$Ofs>x~6G|Y6o(aPa#kk}UbyHzm@&fL{+!7b=dt9lPcyPndr(f8y2{Do| zW|5;~ZsAU&vsd(L%rsbs;V@G30Vs}d~OMA6NwTX33Ob2Jn^oC@Ew0b4u-^QQCJ6c3q z+Aal|&KqSBexw4}_Uh`o6QXS+Fc)qU{qZQkh#B|DZ zc-4HJo&EM}an|0@UdCc0(k!t;lMkwx>G^BO z%#r3BE!3&?0SPu7GnU@YnMZl&6LB_5jvA1|#o2tr7(=96FdHw~S1k&@OqJAJDb%S4 zCo6EOJM--Gb_g`KT>6VfTTu0=N>s5s5}kMQ_q8UqDLgHl-96gCu3T~0ZnbXPRJdSv zjy6%8VRju_SgBIpQckQ8H?=*CKdiXX-B-T&xX#^%UCO3v+p}x)ya1z?S)0ykIq9ly z^Thtd^m!pR4%Qx~J(bs9})BzoFKuo!6WOcE>N82F@LK+xK!fN1vK#eEV6~eSw5Kj(g4t;gq1% zuGCB=?=*Nmd^K@&FfP3?ILO6I5%TRJ{&lO>-10AFYGFRUTX9Jn?~ap6tN!EU25_ru z{SU+=`u`G-Xo3Iz?}$fqH1sUL61?q6SFtf`obP*am|WUI{=%C5@}>DH`1F&8-z(g~ zh6c>8y|^HHrUGj%RddrdJyfk_w1|nT^%o^9JFp78ZF_3IZ#@@|wLJWW zRJtv>_L`^P4+qGfDE0rlpWODbk&$xXez~Ecf#s*n$JnBo-0|`8Hq*(k#>(wn?QKA5 zqmf}mET(B~Ys*MqAIjL<)6L+?mC-jo*cE4TBx?F6Re;Au}+^w^=wx0cj8IzEZaCm5SU@R`) zQ)_>4tG!&4Qr8_yo0t@IPfty4CLJ%d_|wYq@VMo%nlE_fO%`hj>gwvs%gZY($KNQO zd;`isSI1MPIvs0kYgZ?8H!W!DVm9b2^iPvi<>Vr#r?j=T%gV~y!x|fTlQ$PREp8S= z*_h5qEo{$^#?&Qk)V`GL7>FMdgKr!z8aDB?OUX_zz&gnqh!lHztFe^*n)O2@u zmuc=?l02NAiK*-T?ra?q5z&qTlhHVyWhZxLcS-WaFJiR|$4c!!6F8JU@#+%8rj1kdf*!^Lryu&@Wmo12@<%IFP3 zJ=bFOk^o&_*rR_aKvWXx9_%XYzb~=d)2ve1lb9}USA;M(H(y*_>}>^Vbq5ljT@>`m z*k=JujEul2cDCw@CaZM>X=!QcPS)&HcX1eti1=AP5sjhM%5+h?K0C5b&wa@;XD zEz%!I3k^lYXouWx4<*n|g_F-$A_`^%8LKAl>+St!wLYj7%<&rc49`ac;#zOjXx+6N zB$&N@53I@xnl{FtD(;c6NgN z{AjzDWmc{Qs4D=tPCwmj7qUf6lBem~pS1!#zZDQ0fWlH2J5KOwXC!%3Q z;CDFCx7WpLvMcAQ26M2HOw!EBoa^p4JWfHJB|CR995ZS@oaG=Kjty6}>(zxk24aTg zND$pQijxhX;IzNe*VPqXIOlwlXzNgyQ&1owDoRdC8QB=)7=S`9pQlnaSeNcJt~0r? zumIG=D)pExR{}063}jtgTqGrfDYEnPBei)$4qf0uZE`Al(p#5iV?*JXA3*ni4yFK!D($t;szr;s-ZBh=Kp4rVd* zJyohgk(*irgabI{8j4ZMfAGe4Ty1-{PT9uwa6oW*Gl|2|Y1GEXhLN5gY#Pv>^YO)t z7n${d%ys#roaNG}why?`W3}qNbRH)X-_Z7rUTMhusA!9-H-)xW_oSe890AI+;o-mo z2t2Jbvc+J|bCM+j=peNTv##C3#u2}cmDt|PL$a+`r3&ES;K?rM*$3M|fjP-xgLIc) zxwETlb91wfiOur*@(eg+)6>(pdgbfaaodSt9Vt0Eor`<-8OV-!=Uax$`uaLB@o{1j z1iZ>oprx8qRdcQUc3`B}JDPA9cUaA5-J6pWMJAa~yl9JIVftFs#;KOzpu9V8taOrf zD#6~#ChzP+;B-A@uCmRZkN=08vaj!zd1mA7i5(={a}J_=(RL|%XX2E<)&!_|scIh^ z?k%(s5D@fM&_)s6=B-d=>e7CzH?$+e!-O7BA>sK21#1*OjgKo1eNIYB8XlI#kd>7! ztEf0RIUyz{zAy@)@yk+cVMiO9v|luqDXvJnn%ll=y6T|5K2_&Bu2Kw7NrL6bj%1p} zhgDYu);sr!QRzUW9k9Pi-QOo7Y$oT;c(%@3Jo1hm|2E!ZPBNq}|;ik`% zSSy__EHK*ZCazt@YVm<1#A3BV3|i5?6**r1K|x+V>g)t=xAq|D-P&C8?CfkU^u90J z&)sd#DFp?EFP}e0Mn$nQ=}t^c0M+^H^;Mtuz<7lN(c5@3>zwy~9D;=B5C+aJ^ z+Zt%ab#a*`B_%RnBsx;k)A@2xqF4Zy6}1X6;bkWqz>~AE?8e4M;v~C;F6w%?u7nsC zUg{VtccieMhc+s=hw8rz#S9FO$id{YP_iT$tfQqga{Z&q-j9?;?duc#9)#je(@yWmt zfCV1_ir+EO9f@Yk-BW)k!)7O>cGsaIvS}LK}rjkw(mZ7 zl}PJH9~v3N$X{h&!g4F9sXqk!wr=4M*gU}UFBxhQnx;MPQ-Wokb?bzhRO7 zH;ccV>imZBt`g>7TLhK>-dSY$J2QvjH;aGj7{KZFo5j0gn}2N)sBPR?r29KFhw(Ry zzo;U=S-h`J{g)=`fi+KeCINqE>M;Fg@|PTVZ&FTLm>T#)qHkqoW(nNXYny6WQkm-N z0#C~*tt>Q5EP<6grWRB>b{fWJMmkhlrpDAZw{zqEVg+8gbif)XI$)g{J%H(N3^Ma? z-aL>MA4|!T`4Ytipa8h33zRc=0&&dMSNITwp3M*&-m_-BlyEfPzn{oRP?)~w!!RN) z-ZFc>`Qnv%ve0X71TrQim5n3eLHl%k3#+0)36K7i(wFN+S2p$u>y^XWD%jBQKLQ6V z+Pul8V~=}$VJh@t5}C!3Rw10e$tjjXzsey>(6yy|cNIBPU%kM=cwx$lDIFGPCovIG z^qgAyGw1OG`H%LO{P)M%B!Hgok+d%$RtMt_fSbNRwg)v1Ki1^I0z;gRFj+`(%7$lM z`*9GThF5htr)uYI4f`og@Rg2_v}6CJ?g`Up{`?_|8?*82)v?H+E>L7NC^FUnf|97W z&PbFoQt#tf{ECDR#jvP50jsGVsgsQt{bE@=i{S!uy=xl*^ExOlCtR+vjkL6d62@qZ zlv_=ZzzU#;*3rB2q-lWr=_$~<~p-5N#bx1;;qH{oqhTgWfQRk$W!4YG28q*T-Vd|*uO%Bp{NpN^UK5%)jK zbcuig5Ni7ry3v|lGHQ?zp1VWMMpS+JGLwgMm;zZpC!s4hTJ!=PG&ZS2DtjX*Y4*#e zP9uiXe1e92s=a+YwOZ%qYJeK){rYJ}Z%fCPHw>!S zK$g+l34`$1OzJ-tH}OAv72-WU_UwE5w`mn|tCA|66w z${{@dQq4Z-%u#y%rX-|e5CR2g4QeDFA7zk&TN1g@U~kK;^hwb8 z$^pj0pqmoh>x8yUp0Vq(-kW1Pow@UG5>0CHY+&G*#)9%CH=(kVhlj|`pi;B6i<24_ z@NfgIx{YxZ&m(gn{S(#(LVOpR=G>4Vgg+|3(8eY)6M98UuacnT$xaB-^Kqf(D1SKv z1Y~7^r12QWd;26x?-VuWDEl(i{W|k;ik|0T-<;&jv4^}y0uxn zNf*oa(x}-AP)Eo)jU?B?@_GnLK^h9q+^+fHRB`$Cc z7%2!-f^VRfNYt8O-nM!hs|hPIoX|ZZ6<}$X5RhU@>xz%dA;UMU5UM~cd+jV&1da6U z-IO-`%2UWRA=8Iq@XMO~XAEeVABll=2LPs#8{y5oi^N{?`^1Uv-Ys4sM1%&3Scxq$ zF+^BO3m2lg+Kt75_*OiSn-|0G5_Ah*O|JRJ}Msq9Q z9Z;1rI5JOdO^h_Z9-GI}V}(#%gY8PjjCjc7U)JK>tejdo=xoLA>lQ5#Qr)Mv4GzNGHkNt*G!m@^U=<6o81pv})&)G7!4ULk} zCVNU!xTTQNv-BnMz*l5iF&3^mP8gT+I>>^ulH|}|<-Cs2K5$mP>F-9OB(=R^mvUJ8 zQHj(F56jABEakK~s7IE}l-6jEv8MwY{}X~Bmi5I$5Pu;N|A0vU$e=g})WI|Y;F?c3 z1(LWA1_&vAJ@DM)2gOw)uI~}bVzp?uo1fHV2r8BHe=G*;{Xw4#0H4HO8FO*uX@}PJ zlYQ3pym;JLm+#0R5|D{^7$w!5IxEUD=~@t&4kYYPPq2va}6P2GP)fFXUPiH%C#V?@e!_3$#m>TlBBT z(0iJrv$uUDKKdw8Gn6G+G8*-?EO$Aa2^kdplpJ938C8UIcs788W$#(q=y_C6eGXHt z>`iHbzxt^7JE#vVAM)^J^HdRD^c7J#OkcLguqVg&N@BpLJvS2f3sr2igY(`-$hXz3 z)C;qBqgU?a4?6tKLH(e&KBov&er>Ak8ccw1Cy-zp_XM;H^CLKsL$1 z_;+aZZ(F{*8-#ve&;bCJN3Oa2Kww(of#bh$YH{M3*0$`R^!37XPYjcwI4SGrg{`0C zBZp;+)%AunU6a@(VMhlL&mJ1cheG+we)Z1_ismtf7#PeE1k8zE z7YU8TADj330?H;qq45XY2J^qhC2pY_fq<{B6441o5K3!K>>iXhF>HwlA)6zygny~+ zcpB+CdEFDuMwBV-$TE0V1W}=WejZ}D9S|kXlEbWAGEpiXrIN>LNq@A}I54u3xnC)d zLWq1kGyz9z;H_3|xK5P9-IO|cnKz%`Zb&fydX~fnJasKkeD^e-P7r6fCiv&!nQBmB z{3Nr9>~_A~h|kwdadwSq-zfN}CWx&~=%Q*F@mr$IUXaFKVwn#Fp)tt^~VJ%=;z`J zpKx5gUeY(@TVYk#m@1+;1%OJPY`VQpzoDPoN0MsFFD?CSNxqZAY0dH=68-=92PB`yLWP#5RxotSNI#MFKg_LA-n$9r2hv@mH(t(nQH zuh|E#W_e~$hyC@YX4NaG3%P-es@2+ttgVU6%xoRVJQGduL)aeRE{|^foCAt)_eGv}%j8go z@Jj?p9x$fQ@s%W3e=d^(FM!LW>L5YBQs+kiMu4VX(o9e4jE2j01)8irQa4yB`6#Yt z{aG|8wV5Imrj3HU!iafkWM#OGR%8DQwUGjo34WbQrS{H@b7QDfWP+ab^wmC}4wA`E zvBc1y+70g!81->5;^DkE_s72)%nVZ1fwXndJ3j>Q0jfjC`0sO4f^knVOd}1l*SrEt zI{cfk%;o@80W+DBPki_lukoIJGT=#`9ZpSQc1mEoVp{O@ZOy};5i|oC;(^XupH!7E z8BCzdUv3QZ+6~3lVADnmb1k3n!3k0lg-(4M7?C7|4C$c#wKk66cxC9Or7Z5Va;$?5 zr`U~-obNN{LQ#aCxz1=NflP*nN^I#LU^R@7O+~L@(ignI411OEidaaTc4AVq-wYCl z;^x}@60}EF%)eHx)iFcb;is%L;26xEenJTuJ9zIU@9Iq9Axc{~)gognT4KvPb`~9=P@O?BS0JM(Gdi%2tMo- zi1^!$8{ou=6q?2r>aanv?Qi^uYc*nLu69SCAE-$4J~U>(aGQr$H4QVfe=m*vx}q@x zZ|Y?Lo;d==%)g3aqoqGA-1$G0M-*|}(k8zXah&j72u#i|T?xu07Wzj8l ze_ihu$+kBXL@3nL*$_Fivl%g`3hkfz$QjnVx3Sly@TKgcw*%12`@_|}WNe@~U&`Cy zZ4igyDAdc!c+)W8Vcs5|nAAfN_$&7S&^@Mv>E`B#U&J;%C5`7d}GSh_DvZ z8K!!nuAbGsC(yKw+0gT&o^I?1CE#o`b9?tntBhQC)di!$&j!qV$#6gv9Gav)_kGRz z$+YBnPq>g|?Rdpebc%$M_DkM!1vd1WO7wW;>-iTlovhDJc-fU0uDH8U8RtQsg$VdO zfaLDB>fh%j9phuH`g2&ZL<`ORSf(&b7>)hcS~>9gcCDQ26F>I^Q^n~Q5l3P>-YfU1 zn76>u#H$6MLy8+{N7B@Lm#*&bBWl7~Zri}vjj`N_} zX=mu2`ie8Fg4BM`{a&$A?SYS(xLCNks&yrw!fpPfY_fx2w3J{%8ta&(AL^nOWFLZ$ zGR)#akbnL7c-gz)j5wn*AmM5(L}&~jrDO7JG-5Oqd@R=5~hL`{&~X(z9kY zjTp;B&E_QUl(w+fn$u$vhbK8Pt7-Q9K7b|W5KfF0aI(@L*7y3KyRLZU8u+Q$+NStX zuqhUY4hg6Ce0VcrgmE>8(4RKUXKP<^_QEh~mNm<(j*kd2s#+X%w2flHNS#^Xz>Lia z^cr%3`Jq93h~PZN+Y|eKkpr0H%g!tz7w~+4WLhD8`H<@Y&w6@^$|8jTNrZo?eReuC zLK&{LZye)5bTn8jUERMW+iR;SnJrB`v@=@h#5U+c#ll;Q{t*PkxyJB?#i4H&RA##r zvcGhuB=u{xwibNj*=6Qa_<7rjSL0x#(^)74=-{Zd@Am~tQnjx6V6sLh%ahJYIU?mO zluw?JeHOiZ2;^Ux4(+2VO`ue<1RqcVK0vv4Ul{*1Bb4>@1+Qj$r#E`e7(4pf?aU;y zPYD{4N$OoKwT%9Wf@A30eEO?VQp@DxQfSryKd*e z=FNx_OYO+&CX|w0w13I$3*KW+>TuWa*=@y_b{7QwRc#nH&Wd4(&O;#ioCF%MA!>cc zdK;|`5~aZmwVvw8r812VMdbl{^uNRZDi`-3;eWdIXSU92KZ+0-w+iwKa`;Lk`-N)& z8g`%Z3>7Mj(rBeAwRNPg7(J!@LRSU`fI5jbvawEPuaI2{P$$8p5AHLfMKAb=PSW;+ z-sK^He`QRxw2v&530}cGqD|b$o;RIX%0imT=?1|gekATNj-@!GJsIzWnAUehQ*UO) zi;q+Kn>;jTO-Oux-&#-aTCmlm)S!>Q;= zD7MP{pwTajNfGJRM+Q&4uS(@EisD*+uAM?+_TaPN4r1VKZ-dS6d`UhBVRTTFcnIt- z%FlmH>dY)mZ49(^EG&N`bU^Lo?-RP;nC9JcJ$_#}0kn^ip!^7(LhoLrH9uAX6md=b39p=q=+F(~D4{lUK? zcOIQH>mwciL!o_u%KZ(^pa#B?24U&|H|2(VHrybWG)M#W` zPl}=>)sA}wN8cw~hN&Cv;>-(KhE(Dg6p=+xf*;K!w>4Uy)P`|URb#CXJ};!B$FBj+ z1P#52evOZM+>&bijf`+#YMTqgN`4~)E~-L*WRS7$3E~fij}Jh(d$`Q+Gn4+&nHd;f zEYVB@=Y1;?q!1ct{uiZl+x&b>>H7LiYiyBcq-V?m7n|oyZ`Qw6Pi2eZFGRaHOX;l9 zjnFuI=bjwzAxXmwXX9{uWOy|$ezq3T~2gc}%5V9|^2)^o7EN_ihTy`TE4cPM1 z^(RV*%N~4Nnyn~Ssi=Zr`m!P0iw>=ztgetjJEeC^&v|u4$KjM%CxByVD^i+{e1-c+ z7JZDm@>&Je>}jC!5*Sur=XXo!h41u}7^8`F1qEbh*pcv~cuO_iutp1N7L!~cHB@W} zKRrvL7Q^8Vq0jbMbhJ%XK7|zu??aJ}*Tu1*x=%#Qxcozo5x+33Ru)Tu2pD#G#e`jS)+z^xf+V1f z@h~1NY#~~14H(fBesIcGuWK@dkJM)_nM{qN2|lMZ49I*d8v?sTQP=#chFA(xo?k#6 zP(0{EYV{qhyNLCCq(Rb#hBQ63qjgy`xCoT;j85R`kKK{o4zbVF#2BYD!yGpCSh6?c z4~_T(eDH5=F8?@J9tWN)o7gad6eI8#$qasqBvd+u0iG+LvjTKm%7z(3p|zAuRtH?W;2Z3S83%$S$Dm$$k7ecNt0U zYtB>NIYCJD_iG0y3Bg|ZepqLkb7u=bp=xl#_>%~uK4+G6O0C$t_(Sg5*9{^O)nxiX1BgPdaNSg9Yan~ufJvgcy1Kh*RecBfqgSnZ>o(S z%uK0qjgBwNzX(J!Vb#@9W1XzP@O8}eKQ&LSH*BVZLt8tp!JYetyC|akX;~dsG2m^C zv`M1JfwxUup*wXm%`SjxE-%RZ0!L#0Is3GX(?d~tfc7yyqKZvg2NnbKG#vkllUIGp zxUjPcVe?1Nka6w_gr{Vrcl92NSrl;X9@{mmmcW__Xx=R%?a&ursCcv4_04lnm_A9x zW+6qveZ{{CcnTCX>wv`(qB>%ks08t8yaWUqjV(Bh8wl~lpT@uQU6gP3h6^sWy9e1N_W2n7c`2Y6fEX19X~tw+saASF}vX zzJ17-S0E5(XO&{ys~yw!V%YSD3^K+PZQm-`#K zp{}Wgv4$0;rL~!v>F>fJJ@9cNioHo!Fa-@|C_BOdupQ~@=ckCXF6LK;Qc-^ra^DxhnQ zs4ciOxPpb_y|KiUi`$J1GZZy7R`JBN{Qdc6oUny}!l6BPN&Nn4iHlh7#@8Kaxkv`g zheG^UhDA&JSPBaa;gA3ao?Ki2rEt5YHk^7P-~01A{2U@TP#6+suEniaBojgtTfO7g zujdI=PC~R6ItN6`R3wkEE*xv^uuatg(c`OOGES8o@X6GBd(w261m4HRR4N1XoxQGzb~4TU$+>i{ywptKdhmyT=Ua6g0Y zz`(+SC>!LHL~8F?=_aa!I*5{a4ECbTamLt}1Stl(n+HRB_H}+_WuaKRK}yAk<+^Cj z#guT-y}bx2xo)V3HnRa~#|LEu0WrGP9F3QA`<>f{os2q0TLsJn!7dsk1FfrL4xN?F$5cz~&Zf$nWjcbH)vQj!tD zrlFsRg5Kt_S}}jED;H$G(CD+xRa@Ysl2j*@i*7EA0bwd-m1J5d(Eom&K{$A}-b5z1 zN^G_pjjNiqC5Vy7@%%}N74G+ky#FiJp?f@^hKY-|(ZIFtULaqAm5NVkgX{rSJn;!} z_RASojy0a=E-uO12Z8tj4f^r)SIhQ;b=J{Z(Y(#k?yXWjsl){uPS5oEwwDPqkXu_8 z0n@6c{N?oNET|H=13I_nP8ze zo~gf^wwr&q-_m4v04Zar&VbvX9q}eLpd_dn7H=50U-ez48Y%V;4z`)**hf95S-u`c zB7D-8n^Ba79xcQ^*!_0-TnRScmyTo(=xG+|7__%8#?3=`Z zOjI%c4>uEnpT`srD{Tzr-?eRq>~{o;ZSj2|nYLIYFuGH(w)rg|S zI?2?w&|LFXUdYg)Y3AMU;PZ&S?w*?M71qZ{8#(uqeoH)FIHCK-g*s@`64T*Vjrv-( zJWesGTz{;m|6p;p!Pq@7rV?HyG4exP{KRGqCdTt5>}NY2Hp8CZ+zh~V&14t7wC&IJ z3vE$7;XlK4xQdZ9dsPJptBfnrUyO zSun+|sqj<0h`}N1%XBnq?>r1KzJkL;A^Okg8sPRs+UxeTy`SD745AshZx&yFO(o)r zZPnj!5bu{R^g_+J&^=#r=2o=A@q)#j+q zlzi$mD-VejK8i=+C4Cbo_@*!UdHCCm$XHf_`40C6JfnUrt=|;Mc6j=S_S1Jy3H?1G z10E|UNU-h!a&J6zfq8{0506$6gU|Bv>Hgubjaosr)mx;BxW!bQY|_{E;iBnZ2@c9d?JH%e`n#bhp3ghAp2Ag8K~*4(cHJQvL&`X%0#aF$VFbdkun z)SI8VZV$2CyC=5(G2H*L+Gwm}1+3KiU26nBy7ZO<(lPwEIpE!sY5!>kKDJ?)ye(J- z)@i)~){UP^(!^`O?tpJcb*C679r_d*proJbwQ5@wt>Db=Fz_|>qoBK#XLQ6sQq0cP zo*V5Xim+miWjBg&=++mEwxc=wIkQRap#l}c{dQi}T1OihcxcROyTHLs3lqml2G)MG zcStx76|3AmO81}qW_oma4*KPwI6gyM?a!*!pCRnGxnY-ybTGB*BuOP~)?f^Y_t+-| z%&&F<`W$xjxuXjR^f~PU+-j5S>GXUFDMffnntXOdISI6gd5cm;$Ey_+8}~>^M7paE zfjC+nz>Po-u^jQ3PRPI~i-SDH^l_xIOd=@hctM|V4@9C5sT4kZ8>;3!+FLcV>+n6X ztJ~$cL*Ks0oP)f9Vth!Izk6ozKj8&Dc9?4Yspj^Opa0?lGJ3Y zG=XRQ7tf};zPnkznJpr9VrGsGo+ zx!+=Ph4;^qI-J-@@qp*7H{b#%J*%lv*+?>x!3wWvi{QZDd&Ql_u5S-i7XUH_9w%1Q z&hcZB61fqEqhLH+EOuh=yAYqDNDcT-Uo>bp$%4PDVI^0l)_9pfnqNZtl1jeDq1ENv zb`9mqOE^Sv*Ld`A21)cEN8+2`*shT7*pWcatsN zcIH*yX5bbePo5@Y@jT@7Lu}-+`E|=i`0wuexdPtx6PFQ*&?F4?>Dllzz;{PVxM|># zXz&yp1f#Z`h-#bpOU|ZZAQ=%VmX;NTWDML!E}Xq+vlS!#T0Be2IjibdSaE8VI8q(| zEH_ikeJJ-6is%oz@n`5+ULE_x<0efQFU8D{+~GAXPHC2(jmK7sM8W8 z%r^{K+J9b_V^t((&OIo^w3yeW^GY$bk=qoKZ`kOuthp(*qJqrdPyXEJ&wS!%?(RE6 zOh=WQ3*EP;>dTZ0Zjax~&aP#}TSQ;)eF0-W)yQ`W@|j*?;GsZ0z)5HyTh^>U-B#Nm z2wbqVaPcIF(T2nMwz%JwXM|X%d!c@qKU3XaM%(4!?4hUk`=Nu_D8}|5mRU}tYsH44JUgT1JbEGC za%$-7fE%ab=BvnuApBJ%qNRPT#3WFP?RKoUEMb!0n`61E5%RlvhOBX~BaBRe!H}f< zxD+dab%%fFhZU^XYU3a|MyaF8pV^tA%;}3+&Qxu;UYHy~S6*zE=`&@yzb6(PZ8+QH zY-f!`RrTX2Js5%7B&kW0Pk_ilK<6O9Mc^a|z1`nT&x+qH3(Nl;0p8o)3(wCd&i^)mg^?;dQ@LqMfS(&(X4+6iI3VDLR(XMm-|qR{>ondM*VV6>TRK?m&V)| zW=W}0lMfZ1Dm=VGW+ANZS|6W)xc@)JU1wBONw-z94Jbj7phyx#W#}A8PLm``6p1Pz zK?IZ}K~Vt_L`2D1CFjtJ1PKa~B`PR3B0-`csHg}n`0mATW=;3)^)YX~ACF(Gy-s!2 zt#r>lb!zYFP^-|VQJL$zKlcTj5`(@bJ>{#m#}eO?sv?Kjjkp}kBxmDmy~~2;tT-uM zrPF>5Hc5q%hh8EoP6iPYa(;<*obi$?d1YN|@lDZl?*5*-ts+?T*fKvj2!l(D+Ko-rVl~l`(A(lT)n>gkFk;v z%?RFh9{cv1ER~PJZtqqpyP7p!m*ii5wh6dY|6-|l=dkV}=~pqUXU$O_XEZEK?%b3T zO3yR(_wLFWhJCQTXgA|oT1aWtAD&cP&AoXyU8e2WvS%E8+%>OgMx@eFq37nr4ZOg`+8KHgB_R5OA_mxx#ukG>V9z{14Ba@3Q!WrADoM|CP< zW{W!iu=gL+yPon@sJ^ZIw6lKe-lw)h^hTP_IpzN{CY)1UF1kI4%}pEUZn!$P@qWt3 zhfCghzfg1Sh}k2q8v)|xU(yE53XeJaB%h>wR1YDmp(f25CPN3$t!MYJ@8LhtQm10D zH9}aHlGdxoQ)9XelP^#=o;e>CGr8>t7EN-{3Cj!y(;nJ~xU{VCpW9Pyq+NO$dYZi8 zM)9o`oOApa#yH#dp#$kXAf4#gX0AhO@oj|uGdNesuMXjxu@%m|Y??QcfTZxzo}Juy%dt@K3eT7kaWn=NhYT=Kgh_VhzsT>-paRsFj~-| zLx?p(W_xByT2rb_*Auc-oWmJg1EEWz-w;0s)){(&+*Rpx&xU{j^VHUZcJy)?#N`Cz3CNL;xKJ`yU}FYBetr@@0o_z zuRyY5k?Xk)EnqM7o7kWdb3@=81RsR7ZrE| z<1+E>uw)F=Us<6(1lexz3dT3!?<@aIF!wfoI#hzJA+Rw{`R?L4e*fK|L!Z{9kz~VqEhp;cG#>{D}dQQ!rjT+v$*Q zvqWiq_R`+QMW1*dL_Dh)ymOvI=LgJ3N%BUjmTt3=@`U=Txtns!^lizwdOi7$*)fkF zmlLLk9yA*BZ?=RHJ@>y&_SKL&H3 zTTiRlm{cDpehe!7CQ?&&GplXMvD-tXtd$TaV8+F`w6^N~W^($Ay>;FgzMm(ha7)dj zZ^gs4^QBLO4K4LK*<|))Ix@PQI?_M%Z0(oZWu?l!l{rW6*59&=C~?2EHo-Hc?^YlB zPN`(%-8L>JoqX>5DaJE`6CRqrUyLWX7pD_1vQ0|{GDPQV1})6a3;MTMKiH+rDKkrv zK94@tB~;}Sp%M|8$u;(!+oAmww>o@6FUQ|PQqtMfUd=X$w~5g#3)WEe9^Sn!xC<$! z*O1zAr{>c&gP?bX5tL5s(NhHno>;rQ?_+x%A;J1GgHr+-F~`fm+;Td>JM{{a6A#>! z?-zW&ERv}-CAZJUr1%g@L_)zegal8wG`H04N!jwPfMxI(`9d>#&y$@6&a8WG)ptG2 z+THueW$*5uE7l2%&-b=fX3{(R*$_aX*;|mCrhfdsmHj(4yQn7Rbo5m-Ht|83*{1Hi z5Pl2a%ZHylEQ_%Tw+Y{M*Je1Fm@5!2fq-$Y1vb0B9VxqZIWW;8GoCEEy?v6^OKE~M}fELxUp(#P1< zhn${y`Y|s}1nz!qMgFLB<;O#o_8KyJ?h3yRYpB;?IBOoidhBZO?)^VB9yKw)n%fe_ zedKG|_U(TP@=lcGwtnSw?zK##*Hd&HM$Bg0ZGaw_x6LsO(k7fGmrHa0eaZC1?|20B z5k&i))Z>k)BFnI*cTwmSUH-2AJS8XhxneV}n1C&(l5<{PP3pSI&x4+C|w!iP_z4u~4)i5UG)~|BI8;O~NmZGBVVs5)Q4gMeUA|PE;eRlv;E8I$76hYp8NHGDhj&Lm zrf75i%zwWivq3NN&)miT!C&$bck z{8!8-R2OA_D?QQ7icQb-HWDj|9M^1~03;c)6*WWkQcN2kb>2+oe_q|vup-t2xJGr zg|Ck1k|^RScgR;drP3@r8y#W;{1`b5;BY!4{ac`lrul?TlO*&>&Z-Emp*0|*GC!cg zyT?GDN&CKKaAmJp)pBzBp5*kAy(`(Ot+se}Eot{Lh53t`JS)+O;n}s%Z+?~YtNO9w z{kVC=z8~9Vb$H^d8B4T=@wuxwV-q~GK%<*)ksoo1^Z>C;=+`bFp~s!ybmX8j+BlB{ z+v-XuIcpD)ERx<#%ph1q`)~_M@zXsQ$^s1TmB{<#SGCA{3a1Lb1n3d-b zC@y9ml?=vg)tx%8%)v0m3gNydgMvDq0KbOgs&d_loMc!@a)ur zx!LA~@828?FtU~%kNV5TJ+pLW<`TPRpAAim9~ODhnqYC-*fd=wMfkX4b9=#!LS~KC z4==wgX!*;U2kUG1REYJ}$-Ma9q|Lmf(cFZ6ujJ4rR>s$~zAsXVb-=T!T`EibfS$_i zpLv8K=YeJhBZTS%s%3nc)jTUnvOC)TTGss+f>--uS)%(+e&T5h3?mX)Ix9)iBg8FfXpy=z9e#YUVsx%H!{Y|A1ByM47=V+}uhFJ(42$Mi&Y5X#<164v`d~h_ zcjqOqyVe3HxDgov_Ookj*zy?VFuUgma1_R$yY`MNQq1EWLl;HTD(G4m`9+RjKlTBWVi32<+-NWNc~lw8ANoxGoHJY9E;Kxgp!j|lKvTZjx82er#p^x94EqNKJqPn1t2||V}5xj zA#g$NKWP>&()?fE=5Dyij+tTEOpT|JT2c!O@dMx(3dw|^wNQ&mA8zF&TnI1YlTvxT7&S<9Ml|#tme7QtG=@Nx`@V)-%bsY`WS4SZce75{w0hnDYXv9XdE- zV<|1&<+hd&T<7z~R9cu>OlM&JF0kjlloE9&!}G50c_|}~b~3yUpI7cV60`!9NR^NK z23^}*-AcbYv6>FJ4zs(Ejvk`h4|H5C%w{eNc(ojGS(qrbjdFNf1@q~@!?MidJp1oo z4O)JmnfcUmP-IbF@!4aRoEclBn3g=7$IgNs7H`RGdYvW;oV)$h%8~04K%(uhN-z&| zr%s7ef#d9psNVFP7qvYYl_-0U+gJN^-M6OH=)R39#-|;U@2W4%m6|>2Q^}E3?HKNv zyI$O2n){eNMdzwVcpP$yru8KZ;gJfW|Ly#ov`ihvVf@W1~= zu^0@Xo$Da_HwNnCf`6iJe`5%cuaE{qg8ck67#h?QrNOYEawrXk2c<%(F`!itJ0RN+ z*hFdUFd*3j4Tb}yiD@tbc<-1RL&8C^Uuq1E1gSu%F)S9$M5DoQpuQ$GMj(Ka%~TlR zUxG@+G#C;TQKi99px!7Ah5N-#k_b&5bm~Q-LA_~eJ0uo7-lxIvpzJR-hJr)p5YRkO=aj|{3v%bt zV0g&fKmc72DT_ux>qVnM+B)jrLu0|92@OVo%u@sgsD;)Jg92rmsmo$8(Eh{VKrIX! zJ3MqOv2e&dMPO0TKEPr?{yOU4!_w&k90EEgacEGun7S+u3yR6oU^;h@eu4F)_31Ve-3-!yhu$ofD69Ho%H2i6g! z4uJOw($5$yC^k%82ZjJyN5F_f_D&$-89Fxr8evGiSUjk@PF)AER3ZI`!$8a8a45(; zz{63HHUrxg)D8`;1K19rc6jKx;R&F)I(3@~2xuJyG~_*l1eP@Ic%b0W_dW`a0(G^i z>OjG@W8u+fYDG^#A7w^>PX-z@kF930*O^as;dI)21NvX2Q*Sm bT^Urm-k>wtaDPxhrV9?XZ=brp2JC+Tay*sk literal 0 HcmV?d00001 diff --git a/scripts/compare_pdfs.py b/scripts/compare_pdfs.py new file mode 100644 index 0000000..cc5c59b --- /dev/null +++ b/scripts/compare_pdfs.py @@ -0,0 +1,107 @@ +""" +Deep comparison of original vs translated PDF. + +Outputs a detailed diff to identify exactly what format elements were lost. +""" +import fitz +from pathlib import Path +import sys + +original = Path("sample_files/test_corpus/test_pdf.pdf") +translated = Path("sample_files/test_corpus/test_pdf_translated.pdf") + +print(f"Original: {original.stat().st_size:>10} bytes") +print(f"Translated: {translated.stat().st_size:>10} bytes") +print(f"Size diff: {translated.stat().st_size - original.stat().st_size:>10} bytes ({100 * translated.stat().st_size / original.stat().st_size:.1f}%)") +print() + +doc_orig = fitz.open(str(original)) +doc_trans = fitz.open(str(translated)) + +print("=" * 70) +print("PAGE COUNT") +print("=" * 70) +print(f" Original: {len(doc_orig)} pages") +print(f" Translated: {len(doc_trans)} pages") +print() + +print("=" * 70) +print("PER-PAGE COMPARISON") +print("=" * 70) +for i in range(max(len(doc_orig), len(doc_trans))): + o_page = doc_orig[i] if i < len(doc_orig) else None + t_page = doc_trans[i] if i < len(doc_trans) else None + print(f"\n--- Page {i+1} ---") + if o_page: + o_text = len(o_page.get_text()) + o_links = len(o_page.get_links()) + o_images = len(o_page.get_images()) + o_drawings = len(o_page.get_drawings()) + print(f" Original: {o_text:>4} chars, {o_links:>2} links, {o_images:>2} images, {o_drawings:>3} drawings") + if t_page: + t_text = len(t_page.get_text()) + t_links = len(t_page.get_links()) + t_images = len(t_page.get_images()) + t_drawings = len(t_page.get_drawings()) + print(f" Translated: {t_text:>4} chars, {t_links:>2} links, {t_images:>2} images, {t_drawings:>3} drawings") + if o_page and t_page: + print(f" DIFF: text {t_text - o_text:+d}, links {t_links - o_links:+d}, images {t_images - o_images:+d}, drawings {t_drawings - o_drawings:+d}") + +print() +print("=" * 70) +print("HYPERLINK COMPARISON") +print("=" * 70) +for i in range(max(len(doc_orig), len(doc_trans))): + o_page = doc_orig[i] if i < len(doc_orig) else None + t_page = doc_trans[i] if i < len(doc_trans) else None + o_links = o_page.get_links() if o_page else [] + t_links = t_page.get_links() if t_page else [] + if o_links or t_links: + print(f"\n Page {i+1}:") + print(f" Original has {len(o_links)} links:") + for link in o_links: + kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?" + uri_or_page = link.get("uri") or f"page {link.get('page')}" + print(f" - {kind}: {uri_or_page[:80]}") + print(f" Translated has {len(t_links)} links:") + for link in t_links: + kind = "URI" if link.get("uri") else "GOTO" if link.get("page") is not None else "?" + uri_or_page = link.get("uri") or f"page {link.get('page')}" + print(f" - {kind}: {uri_or_page[:80]}") + +print() +print("=" * 70) +print("IMAGE COMPARISON") +print("=" * 70) +total_o_imgs = sum(len(p.get_images()) for p in doc_orig) +total_t_imgs = sum(len(p.get_images()) for p in doc_trans) +print(f" Original: {total_o_imgs} images") +print(f" Translated: {total_t_imgs} images") +if total_o_imgs > 0 and total_t_imgs == 0: + print(" ❌ ALL IMAGES LOST in translation!") + +print() +print("=" * 70) +print("DRAWING COMPARISON (rects, lines, etc.)") +print("=" * 70) +total_o_dr = sum(len(p.get_drawings()) for p in doc_orig) +total_t_dr = sum(len(p.get_drawings()) for p in doc_trans) +print(f" Original: {total_o_dr} drawings") +print(f" Translated: {total_t_dr} drawings") + +print() +print("=" * 70) +print("SAMPLE TEXT — Page 1 of translated") +print("=" * 70) +if len(doc_trans) > 0: + print(doc_trans[0].get_text()[:500]) + +print() +print("=" * 70) +print("SAMPLE TEXT — Page 1 of original") +print("=" * 70) +if len(doc_orig) > 0: + print(doc_orig[0].get_text()[:500]) + +doc_orig.close() +doc_trans.close() diff --git a/scripts/debug_rc_real.py b/scripts/debug_rc_real.py new file mode 100644 index 0000000..1c5905f --- /dev/null +++ b/scripts/debug_rc_real.py @@ -0,0 +1,40 @@ +"""Debug: show actual rc values for the title block in real translation.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Patch _try_insert to log rc values +original_try_insert = pdf_mod.PDFTranslator._try_insert +TITLES_TO_TRACE = {"Spécification technique : Office Translator v3.0", "Technical Specification: Office Translator v3.0"} + +def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align): + rc = page.insert_textbox( + rect, text, + fontsize=fontsize, + fontname=fontname, + fontfile=fontfile, + color=color, + align=align, + overlay=True, + ) + # Only trace the title + is_title = any(t in text for t in ["Spécification technique", "Technical Specification"]) + if is_title: + print(f" [TITLE] size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} -> rc={rc}", flush=True) + return rc + +pdf_mod.PDFTranslator._try_insert = debug_try_insert + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/debug_rc_values.py b/scripts/debug_rc_values.py new file mode 100644 index 0000000..8422b69 --- /dev/null +++ b/scripts/debug_rc_values.py @@ -0,0 +1,37 @@ +"""Debug the actual rc values returned by insert_textbox during real translation.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Monkey-patch _try_insert to print rc values +original_try_insert = pdf_mod.PDFTranslator._try_insert + +def debug_try_insert(self, page, rect, text, fontsize, fontname, fontfile, color, align): + rc = page.insert_textbox( + rect, text, + fontsize=fontsize, + fontname=fontname, + fontfile=fontfile, + color=color, + align=align, + overlay=True, + ) + text_preview = text[:30].replace('\n', '\\n') + print(f" TRY: size={fontsize:.1f} rect={rect.width:.0f}x{rect.height:.0f} text={text_preview!r} -> rc={rc}", flush=True) + return rc + +pdf_mod.PDFTranslator._try_insert = debug_try_insert + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/debug_smart_fit.py b/scripts/debug_smart_fit.py new file mode 100644 index 0000000..06319f4 --- /dev/null +++ b/scripts/debug_smart_fit.py @@ -0,0 +1,81 @@ +"""Debug the smart-fit behavior on a real PDF.""" +import sys +sys.path.insert(0, '.') +import importlib.util + +spec = importlib.util.spec_from_file_location("pdf_mod", "translators/pdf_translator.py") +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +import fitz + +# Open the test PDF +doc = fitz.open("sample_files/test_corpus/test_pdf.pdf") +page = doc[0] + +# Extract blocks +translator = pdf_mod.PDFTranslator() +raw_blocks = translator._extract_text_blocks(page) +merged = translator._merge_adjacent_blocks(raw_blocks, page.rect) + +# Show block info +for i, block in enumerate(merged[:5]): + print(f"Block {i}:") + print(f" bbox: {block['bbox']}") + print(f" font_size: {block['font_size']}") + print(f" text: {block['text'][:50]!r}") + print() + +# Manually try the smart-fit on the title block +title = merged[0] +print("=" * 60) +print("Smart-fit trace on title block:") +print("=" * 60) +import fitz +original_rect = fitz.Rect(title["bbox"]) +page_rect = page.rect +print(f" original_rect: {original_rect}") +print(f" page_rect: {page_rect}") +print(f" page width: {page_rect.x1 - page_rect.x0}") +print(f" original width: {original_rect.width}, height: {original_rect.height}") + +# Simulate the expansion +margin = 18 +max_x1 = page_rect.x1 - margin +expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + max_x1, + original_rect.y1, +) +print(f" expanded_h: {expanded_h}, width: {expanded_h.width}") + +next_block_y = page_rect.y1 - margin +max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * pdf_mod.MAX_VERTICAL_EXPANSION, +) +expanded_v = fitz.Rect( + expanded_h.x0, + expanded_h.y0, + expanded_h.x1, + expanded_h.y1 + max_expand_y, +) +print(f" expanded_v: {expanded_v}, width: {expanded_v.width}, height: {expanded_v.height}") + +# Try the actual insert +from fitz import TEXT_ALIGN_LEFT +french_text = "Spécification technique : Office Translator v3.0" +rc = page.insert_textbox( + expanded_v, + french_text, + fontsize=title["font_size"], + fontname="helv", + color=(0.1, 0.2, 0.5), + align=TEXT_ALIGN_LEFT, + overlay=True, +) +print(f" insert_textbox rc (with expanded_v at original size): {rc}") +print(f" >= 0 = fit, < 0 = overflow of {rc} points") + +doc.close() diff --git a/scripts/debug_smart_fit_trace.py b/scripts/debug_smart_fit_trace.py new file mode 100644 index 0000000..c2daa35 --- /dev/null +++ b/scripts/debug_smart_fit_trace.py @@ -0,0 +1,55 @@ +"""Debug trace the smart-fit behavior on a real PDF.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +original_write = pdf_mod.PDFTranslator._write_translated_block + +def debug_write(self, page, block, font_path, is_rtl): + target_size = block['font_size'] + original_rect = fitz.Rect(block['bbox']) + page_rect = page.rect + margin = 18 + is_heading = target_size >= 14 + if is_heading: + expanded_h = original_rect + else: + expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + page_rect.x1 - margin, + original_rect.y1, + ) + next_block_y = page_rect.y1 - margin + max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * 3.0, + ) + expanded_v = fitz.Rect( + expanded_h.x0, + expanded_h.y0, + expanded_h.x1, + expanded_h.y1 + max_expand_y, + ) + text = block['translated'][:40] + is_heading_str = "HEAD" if is_heading else "BODY" + print(f" [{is_heading_str}] text={text!r} | orig={original_rect.width:.0f}x{original_rect.height:.0f} | exp_v={expanded_v.width:.0f}x{expanded_v.height:.0f} | size={target_size}", flush=True) + result = original_write(self, page, block, font_path, is_rtl) + print(f" result: {result}", flush=True) + return result + +pdf_mod.PDFTranslator._write_translated_block = debug_write + +PDFTranslator = pdf_mod.PDFTranslator +PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_translated.pdf'), + target_language='fr', + source_language='en', +) diff --git a/scripts/diagnose_pdf_layout.py b/scripts/diagnose_pdf_layout.py new file mode 100644 index 0000000..f4b25a9 --- /dev/null +++ b/scripts/diagnose_pdf_layout.py @@ -0,0 +1,71 @@ +""" +Deeper investigation: where exactly is the text on each page? + +Compares bounding boxes of the original text vs translated text. +If they're in totally different positions, the layout is broken. +""" +import fitz + +orig = fitz.open("sample_files/test_corpus/test_pdf.pdf") +trans = fitz.open("sample_files/test_corpus/test_pdf_translated.pdf") + +for i in range(min(3, len(orig), len(trans))): + o = orig[i] + t = trans[i] + print(f"\n{'=' * 70}") + print(f"PAGE {i+1}") + print('=' * 70) + + # Get text blocks with positions for original + print(f"\n--- Original (page {i+1}) text blocks ---") + o_blocks = o.get_text("blocks") + for j, b in enumerate(o_blocks[:10]): + x0, y0, x1, y1, text, *_ = b + text_preview = text[:50].replace("\n", " ") + print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'") + + print(f"\n--- Translated (page {i+1}) text blocks ---") + t_blocks = t.get_text("blocks") + for j, b in enumerate(t_blocks[:10]): + x0, y0, x1, y1, text, *_ = b + text_preview = text[:50].replace("\n", " ") + print(f" [{j}] ({x0:.0f},{y0:.0f})-({x1:.0f},{y1:.0f}) '{text_preview}'") + + # Check fonts + print(f"\n--- Fonts (page {i+1}) ---") + o_fonts = set() + t_fonts = set() + for blk in o_blocks: + for line in blk[4].split("\n") if blk[4] else []: + pass + # Get fonts from the dict format + o_dict = o.get_text("dict") + t_dict = t.get_text("dict") + for block in o_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}") + for block in t_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_fonts.add(f"{span.get('font', '?')}@{span.get('size', 0):.1f}") + print(f" Original fonts: {sorted(o_fonts)[:5]}") + print(f" Translated fonts: {sorted(t_fonts)[:5]}") + + # Check colors used + print(f"\n--- Colors (page {i+1}) ---") + o_colors = set() + t_colors = set() + for block in o_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_colors.add(span.get("color", 0)) + for block in t_dict.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_colors.add(span.get("color", 0)) + print(f" Original colors: {sorted(o_colors)[:5]}") + print(f" Translated colors: {sorted(t_colors)[:5]}") + +orig.close() +trans.close() diff --git a/scripts/rerun_translation.py b/scripts/rerun_translation.py new file mode 100644 index 0000000..2e55773 --- /dev/null +++ b/scripts/rerun_translation.py @@ -0,0 +1,33 @@ +"""Re-run translation with the new B3.5 code and save to disk.""" +import sys +sys.path.insert(0, '.') +import shutil +from pathlib import Path +import fitz + +src = Path('sample_files/test_corpus/test_pdf.pdf') +dst = Path('sample_files/test_corpus/test_pdf_translated.pdf') + +from translators.pdf_translator import PDFTranslator +PDFTranslator().translate_file(src, dst, target_language='fr', source_language='en') + +o = fitz.open(src) +n = fitz.open(dst) +print(f'Original: {len(o)} pages, {src.stat().st_size} bytes') +print(f'Translated: {len(n)} pages, {dst.stat().st_size} bytes') +print() +print('Page 1 font sizes:') +o_sizes = set() +n_sizes = set() +for b in o[0].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + o_sizes.add(round(s.get("size", 0), 1)) +for b in n[0].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + n_sizes.add(round(s.get("size", 0), 1)) +print(' Original: ', sorted(o_sizes)) +print(' Translated:', sorted(n_sizes)) +o.close() +n.close() diff --git a/scripts/test_full_pipeline.py b/scripts/test_full_pipeline.py new file mode 100644 index 0000000..f3bb31c --- /dev/null +++ b/scripts/test_full_pipeline.py @@ -0,0 +1,72 @@ +"""Test if the full pipeline (extract, redact, re-insert links) affects insert_textbox.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Step 1: Extract blocks +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +title = raw_blocks[0] +print(f"Title bbox: {title['bbox']}") + +# Step 2: Simulate redaction +title_rect = fitz.Rect(title['bbox']) +page.add_redact_annot(title_rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) +print("After redaction") + +# Step 3: Simulate link reinsertion (we have TOC links) +links = list(page.get_links()) +print(f"Got {len(links)} links") +for link in links: + from_rect = link.get("from") + if link.get("page") is not None: + page.insert_link({ + "kind": fitz.LINK_GOTO, + "from": from_rect, + "page": link["page"], + "to": fitz.Point(72, 72), + }) + elif link.get("uri"): + page.insert_link({ + "kind": fitz.LINK_URI, + "from": from_rect, + "uri": link["uri"], + }) +print("After link reinsertion") + +# Step 4: NOW try insert_textbox +french = "Spécification technique : Office Translator v3.0" +rect = fitz.Rect(title['bbox']) +expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3) + +# Test on original bbox +rc1 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After full pipeline, original rect: rc={rc1}") + +# Test on expanded_v +rc2 = page.insert_textbox( + expanded_v, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After full pipeline, expanded_v: rc={rc2}") + +doc.close() diff --git a/scripts/test_merged_block.py b/scripts/test_merged_block.py new file mode 100644 index 0000000..8767e99 --- /dev/null +++ b/scripts/test_merged_block.py @@ -0,0 +1,87 @@ +"""Test if merged blocks (TOC) fit in expanded_v.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Extract and merge +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +merged = pdf_mod.PDFTranslator()._merge_adjacent_blocks(raw_blocks, page.rect) + +# Find the TOC block (block 3 in our trace) +toc = merged[3] +print(f"TOC bbox: {toc['bbox']}") +print(f"TOC font_size: {toc['font_size']}") +print(f"TOC text length: {len(toc['text'])}") +print(f"TOC text preview: {toc['text'][:80]!r}") + +# Redact +toc_rect = fitz.Rect(toc['bbox']) +page.add_redact_annot(toc_rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) + +# Test insert with expanded_v +margin = 18 +page_rect = page.rect +expanded_h = fitz.Rect( + max(toc_rect.x0, page_rect.x0 + margin), + toc_rect.y0, + page_rect.x1 - margin, + toc_rect.y1, +) +max_expand_y = min( + page_rect.y1 - margin - toc_rect.y1, + toc_rect.height * 3.0, +) +expanded_v = fitz.Rect( + expanded_h.x0, expanded_h.y0, expanded_h.x1, + expanded_h.y1 + max_expand_y, +) +print(f"expanded_v: {expanded_v.width}x{expanded_v.height}") + +# Translate the TOC text +french_toc = """1. Introduction +.......... 1 +2. Installation et configuration +.......... 2 +3. Moteur de traduction +.......... 3 +4. Préservation des formats +.......... 4 +5. Assurance qualité +.......... 5 +6. Performances et mise à l'échelle +.......... 6 +7. Référence API +.......... 7 +8. Dépannage +.......... 8""" + +# Test on original +rc1 = page.insert_textbox( + toc_rect, french_toc, + fontsize=12, + fontfile='C:/Windows/Fonts/arial.ttf', + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"Original rect: rc={rc1}") + +# Test on expanded_v +rc2 = page.insert_textbox( + expanded_v, french_toc, + fontsize=12, + fontfile='C:/Windows/Fonts/arial.ttf', + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"expanded_v: rc={rc2}") + +doc.close() diff --git a/scripts/test_smart_fit_no_translation.py b/scripts/test_smart_fit_no_translation.py new file mode 100644 index 0000000..2d3e758 --- /dev/null +++ b/scripts/test_smart_fit_no_translation.py @@ -0,0 +1,46 @@ +"""Test smart-fit with mock translation (same length) to isolate the issue.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +# Mock the translation to return the original text (no length change) +def mock_translate(self, text, target_lang, source_lang): + return text # No change + +PDFTranslator = pdf_mod.PDFTranslator +with patch.object(PDFTranslator, '_translate_single', mock_translate): + PDFTranslator().translate_file( + Path('sample_files/test_corpus/test_pdf.pdf'), + Path('sample_files/test_corpus/test_pdf_no_translation.pdf'), + target_language='fr', + source_language='en', + ) + +# Check fonts +o = fitz.open('sample_files/test_corpus/test_pdf.pdf') +n = fitz.open('sample_files/test_corpus/test_pdf_no_translation.pdf') +print(f"=== Mock translation (no length change) ===") +print(f"Original: {len(o)} pages, {o.page_count} blocks") +print(f"Translated: {len(n)} pages") +for i in [0]: + o_sizes = set() + n_sizes = set() + for b in o[i].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + o_sizes.add(round(s.get("size", 0), 1)) + for b in n[i].get_text("dict").get("blocks", []): + for l in b.get("lines", []): + for s in l.get("spans", []): + n_sizes.add(round(s.get("size", 0), 1)) + print(f" Page 1 Original: {sorted(o_sizes)}") + print(f" Page 1 Translated: {sorted(n_sizes)}") +o.close() +n.close() diff --git a/scripts/test_with_redaction.py b/scripts/test_with_redaction.py new file mode 100644 index 0000000..15b8b2c --- /dev/null +++ b/scripts/test_with_redaction.py @@ -0,0 +1,59 @@ +"""Test if redaction affects insert_textbox behavior.""" +import sys +sys.path.insert(0, '.') +import importlib.util +import fitz + +spec = importlib.util.spec_from_file_location('pdf_mod', 'translators/pdf_translator.py') +pdf_mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pdf_mod) + +doc = fitz.open('sample_files/test_corpus/test_pdf.pdf') +page = doc[0] + +# Get the title block info +raw_blocks = pdf_mod.PDFTranslator()._extract_text_blocks(page) +title = raw_blocks[0] +print(f"Title bbox: {title['bbox']}") +print(f"Title text: {title['text']!r}") + +# Test 1: insert_textbox on fresh page +french = "Spécification technique : Office Translator v3.0" +rect = fitz.Rect(title['bbox']) +rc1 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"Fresh page: rc={rc1} (negative = overflow, positive = fit)") + +# Test 2: Apply redaction, then insert_textbox +page.add_redact_annot(rect, fill=(1, 1, 1)) +page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) + +rc2 = page.insert_textbox( + rect, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After redaction: rc={rc2}") + +# Test 3: With expanded_v +expanded_v = fitz.Rect(rect.x0, rect.y0, rect.x1, rect.y1 + rect.height * 3) +rc3 = page.insert_textbox( + expanded_v, french, + fontsize=22, + fontfile='C:/Windows/Fonts/arial.ttf', + color=(0.1, 0.2, 0.5), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, +) +print(f"After redaction, expanded_v ({expanded_v.width}x{expanded_v.height}): rc={rc3}") + +doc.close() diff --git a/scripts/verify_b3_5_fix.py b/scripts/verify_b3_5_fix.py new file mode 100644 index 0000000..9607e7a --- /dev/null +++ b/scripts/verify_b3_5_fix.py @@ -0,0 +1,62 @@ +""" +End-to-end verification: run the smart-fit PDF translator on the +real test PDF and verify the output preserves the font hierarchy. +""" +import fitz +import shutil +import tempfile +from pathlib import Path + +# Create a temp copy of the test PDF +src = Path("sample_files/test_corpus/test_pdf.pdf") +with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) / "test.pdf" + shutil.copy(src, work) + out = Path(tmp) / "out.pdf" + + # Run the translator + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + work, out, + target_language="fr", + source_language="en", + ) + + # Compare fonts + orig = fitz.open(src) + new = fitz.open(out) + + print(f"Original: {len(orig)} pages, {src.stat().st_size} bytes") + print(f"Translated: {len(new)} pages, {out.stat().st_size} bytes") + print() + + print(f"{'Page':<6}{'Original fonts':<40}{'Translated fonts':<40}") + print("=" * 90) + for i in range(min(len(orig), len(new))): + o = orig[i] + t = new[i] + + o_fonts = set() + t_fonts = set() + for block in o.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + o_fonts.add(round(span.get("size", 0), 1)) + for block in t.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + t_fonts.add(round(span.get("size", 0), 1)) + + # Filter out the 6.0pt placeholder + t_fonts_real = {s for s in t_fonts if s > 6.5} + + print(f"{i+1:<6}{sorted(o_fonts)!s:<40}{sorted(t_fonts_real)!s:<40}") + + orig.close() + new.close() + +# Save the translated output for the user to inspect +shutil.copy(out, "sample_files/test_corpus/test_pdf_translated_v2.pdf") +print() +print("Translated output saved to: sample_files/test_corpus/test_pdf_translated_v2.pdf") diff --git a/tests/test_translators/test_b3_5_pdf_smart_fit.py b/tests/test_translators/test_b3_5_pdf_smart_fit.py new file mode 100644 index 0000000..5725c64 --- /dev/null +++ b/tests/test_translators/test_b3_5_pdf_smart_fit.py @@ -0,0 +1,389 @@ +""" +Tests for Track B3.5 — Smart PDF overflow handling. + +Covers: + - Tier 0: original bbox at original size wins if text fits + - Tier 1: expanded horizontal bbox preserves font size + - Tier 2: expanded vertical bbox (3x original height) is tried + - Tier 3: font shrink is limited to 2 steps (0.93, 0.87) + - Tier 4: HEADINGS (>= 14pt) never shrink below 90% of original + - Tier 5: BODY TEXT never shrinks below 75% of original + - Tier 6: graceful failure — block is skipped, metric emitted + - Redaction uses single rect per block (not per sub-bbox) +""" +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import importlib.util + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _load_pdf_translator(): + spec = importlib.util.spec_from_file_location( + "pdf_translator_under_test", + _REPO_ROOT / "translators" / "pdf_translator.py", + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def pdf_mod(): + return _load_pdf_translator() + + +# ============================================================================ +# Constants +# ============================================================================ + +class TestConstants: + def test_font_shrink_factor_mild(self, pdf_mod): + """FONT_SHRINK_FACTOR should be 0.93 (mild), not 0.87 (aggressive).""" + assert pdf_mod.FONT_SHRINK_FACTOR == pytest.approx(0.93, abs=0.01), ( + f"FONT_SHRINK_FACTOR is {pdf_mod.FONT_SHRINK_FACTOR}, " + "should be 0.93 (was 0.87 in B3 — too aggressive, broke hierarchy)" + ) + + def test_vertical_expansion_3x(self, pdf_mod): + """MAX_VERTICAL_EXPANSION should be 3x (was 1.5x in B3).""" + assert pdf_mod.MAX_VERTICAL_EXPANSION == pytest.approx(3.0, abs=0.1) + + def test_heading_min_size_threshold(self, pdf_mod): + assert pdf_mod.HEADING_MIN_SIZE == 14.0 + + def test_heading_min_scale_90pct(self, pdf_mod): + assert pdf_mod.HEADING_MIN_SCALE == pytest.approx(0.90, abs=0.01) + + def test_body_min_scale_75pct(self, pdf_mod): + assert pdf_mod.BODY_MIN_SCALE == pytest.approx(0.75, abs=0.01) + + +# ============================================================================ +# _write_translated_block — tier behavior +# ============================================================================ + +class TestSmartFitTiers: + def _make_block(self, font_size=12, x0=100, y0=100, x1=400, y1=120, text="Hello world"): + return { + "bbox": (x0, y0, x1, y1), + "text": text, + "font_size": font_size, + "color": 0, + "is_bold": False, + "is_italic": False, + "translated": text, + "sub_bboxes": [(x0, y0, x1, y1)], + } + + def _make_page(self, page_w=612, page_h=792): + """Mock page that accepts insert_textbox and returns rc (overflow indicator).""" + page = MagicMock() + import fitz + page.rect = fitz.Rect(0, 0, page_w, page_h) + # By default, accept the insert (return rc=0) + page.insert_textbox = MagicMock(return_value=0) + page.add_redact_annot = MagicMock() + return page + + def test_tier_0_fits_at_original_size(self, pdf_mod): + """If the text fits at original size + original bbox, no shrink.""" + page = self._make_page() + block = self._make_block(font_size=12, text="Short text") + + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Should have succeeded + assert result is True + # First call should be with original size (12) and original bbox + first_call = page.insert_textbox.call_args_list[0] + # First positional arg is the rect + assert first_call[0][0] == block["bbox"] + # Second positional arg is the text + assert first_call[0][1] == "Short text" + # fontsize kwarg + assert first_call[1]["fontsize"] == 12 + + def test_tier_2_uses_expanded_vertical_bbox(self, pdf_mod): + """When original bbox is too small, tier 2 expands vertically.""" + page = self._make_page() + # Force overflow on the original bbox + page.insert_textbox = MagicMock(side_effect=[-5, -5, 0]) + # -5 = overflow, 0 = fits + + block = self._make_block(font_size=12, x0=100, y0=100, x1=400, y1=120) + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is True + # Should have tried multiple times + assert page.insert_textbox.call_count >= 3 + + def test_tier_3_4_limit_to_two_shrink_steps(self, pdf_mod): + """Even on persistent overflow, the shrink is limited.""" + page = self._make_page() + # Always overflow + page.insert_textbox = MagicMock(return_value=-5) + + block = self._make_block(font_size=20, x0=100, y0=100, x1=400, y1=120) + translator = pdf_mod.PDFTranslator() + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Should have failed (returned False) because persistent overflow + assert result is False + # But the number of insert_textbox calls should be bounded + # (not the 8+ attempts of the old aggressive strategy) + assert page.insert_textbox.call_count <= 8 + + def test_heading_not_shrunk_below_90_percent(self, pdf_mod): + """A 22pt heading should never be shrunk below ~19.8pt.""" + page = self._make_page() + call_sizes = [] + # Track every fontsize used + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 # always fit + page.insert_textbox = MagicMock(side_effect=capture) + + # 22pt heading + block = self._make_block(font_size=22, text="X" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # All attempted sizes should be >= 22 * 0.90 = 19.8 + for s in call_sizes: + if s is not None: + assert s >= 22 * 0.90 - 0.1, ( + f"Heading shrunk below 90% (got {s}, min allowed {22 * 0.90})" + ) + + def test_body_not_shrunk_below_75_percent(self, pdf_mod): + """Body text (12pt) should never be shrunk below ~9pt.""" + page = self._make_page() + call_sizes = [] + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 + page.insert_textbox = MagicMock(side_effect=capture) + + block = self._make_block(font_size=12, text="Y" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + for s in call_sizes: + if s is not None: + assert s >= 12 * 0.75 - 0.1, ( + f"Body text shrunk below 75% (got {s}, min allowed {12 * 0.75})" + ) + + def test_graceful_failure_emits_metric(self, pdf_mod): + """If nothing fits, the block is skipped and a metric is recorded.""" + page = self._make_page() + page.insert_textbox = MagicMock(return_value=-5) # always overflow + + # Track metric emissions + metrics_emitted = [] + def fake_record(*args, **kwargs): + metrics_emitted.append((args, kwargs)) + translator = pdf_mod.PDFTranslator() + with patch.object(pdf_mod, "_record_format_loss_metric", fake_record): + block = self._make_block(font_size=12, text="Z" * 1000) + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is False + # A format_loss metric should have been emitted + assert any("text_overflow" in str(call) for call in metrics_emitted), ( + f"No text_overflow metric emitted. Got: {metrics_emitted}" + ) + + def test_successful_render_does_not_emit_loss_metric(self, pdf_mod): + """A block that fits cleanly should NOT trigger format_loss.""" + page = self._make_page() + page.insert_textbox = MagicMock(return_value=0) # always fit + + metrics_emitted = [] + def fake_record(*args, **kwargs): + metrics_emitted.append((args, kwargs)) + translator = pdf_mod.PDFTranslator() + with patch.object(pdf_mod, "_record_format_loss_metric", fake_record): + block = self._make_block(font_size=12, text="Short") + result = translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + assert result is True + # No format_loss should have been emitted + assert metrics_emitted == [], ( + f"Unexpected metric emission on success: {metrics_emitted}" + ) + + def test_min_size_floor_respected(self, pdf_mod): + """The min size floor (MIN_FONT_SIZE) is always respected.""" + page = self._make_page() + call_sizes = [] + def capture(*args, **kwargs): + call_sizes.append(kwargs.get("fontsize", args[2] if len(args) > 2 else None)) + return 0 + page.insert_textbox = MagicMock(side_effect=capture) + + # Tiny font (3pt) — should still be at least MIN_FONT_SIZE (4.5pt) + block = self._make_block(font_size=3, text="Q" * 1000) + translator = pdf_mod.PDFTranslator() + translator._write_translated_block( + page, block, font_path=None, is_rtl=False + ) + # Exclude the placeholder font (used in graceful failure) + real_sizes = [s for s in call_sizes if s is not None and s >= 4.5] + for s in real_sizes: + assert s >= pdf_mod.MIN_FONT_SIZE, ( + f"Font size {s} below absolute floor {pdf_mod.MIN_FONT_SIZE}" + ) + + +# ============================================================================ +# Real PDF comparison +# ============================================================================ + +class TestRealPDFSmartFit: + """Run the actual translator on a real PDF and check the output.""" + + def test_translate_real_pdf_preserves_hyperlinks(self, tmp_path): + """The translated PDF should keep all hyperlinks (B3 fix).""" + import fitz + # Build a minimal PDF in memory + doc = fitz.open() + page = doc.new_page() + page.insert_text((72, 72), "Visit our website for more details.") + page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(72, 65, 280, 80), + "uri": "https://example.com", + }) + + in_path = tmp_path / "input.pdf" + out_path = tmp_path / "output.pdf" + doc.save(str(in_path)) + doc.close() + + # Translate + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + # Verify hyperlinks preserved + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + links = list(out_page.get_links()) + assert len(links) == 1 + assert links[0].get("uri") == "https://example.com" + out_doc.close() + + def test_translate_real_pdf_no_font_below_75pct_for_body(self, tmp_path): + """Body text on a real PDF should not be shrunk below 75% of original. + + Note: the [translation overflow] placeholder font (6.0pt) is + excluded from this check — it's a deliberate marker for skipped + blocks, not a real text font. + """ + import fitz + doc = fitz.open() + page = doc.new_page() + # Real body text at 12pt + page.insert_text((72, 72), "Hello world this is a test.", + fontsize=12) + + in_path = tmp_path / "input.pdf" + out_path = tmp_path / "output.pdf" + doc.save(str(in_path)) + doc.close() + + from translators.pdf_translator import PDFTranslator + translator = PDFTranslator() + translator.translate_file( + in_path, out_path, + target_language="fr", + source_language="en", + ) + + # Check output fonts — none should be below 12 * 0.75 = 9pt for body + out_doc = fitz.open(str(out_path)) + out_page = out_doc[0] + out_fonts = set() + for block in out_page.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + out_fonts.add(span.get("size", 0)) + out_doc.close() + + # Allow some shrinkage for very long text but never below 75%. + # Exclude the placeholder font (6.0pt) which is the overflow marker. + for size in out_fonts: + if size > 0 and size > 6.5: # > 6.5 excludes the 6.0pt placeholder + assert size >= 8.5, ( + f"Body text font size {size}pt is below the 9pt floor " + f"(75% of 12pt)" + ) + + +# ============================================================================ +# Redaction strategy +# ============================================================================ + +class TestSingleRedactionPerBlock: + """Track B3.5: redaction should use a single rect per block, not per sub-bbox.""" + + def test_single_redaction_call_per_block(self, pdf_mod): + """The redaction loop should call add_redact_annot ONCE per block.""" + # Build a fake page + import fitz + page = MagicMock() + page.rect = fitz.Rect(0, 0, 612, 792) + redact_calls = [] + page.add_redact_annot = MagicMock(side_effect=lambda r, **k: redact_calls.append(r)) + page.apply_redactions = MagicMock() + + # 3 blocks, each with 5 sub_bboxes + blocks = [ + { + "translated": "Block 1 text", + "bbox": (50, 50, 400, 70), + "sub_bboxes": [(50, 50, 400, 70)] * 5, + }, + { + "translated": "Block 2 text", + "bbox": (50, 80, 400, 100), + "sub_bboxes": [(50, 80, 400, 100)] * 5, + }, + { + "translated": "Block 3 text", + "bbox": (50, 110, 400, 130), + "sub_bboxes": [(50, 110, 400, 130)] * 5, + }, + ] + + # Simulate the redaction loop from _process_pages_inplace + for block in blocks: + if block.get("translated"): + page.add_redact_annot( + fitz.Rect(block["bbox"]), + fill=(1, 1, 1), + ) + + # 3 blocks → 3 redaction calls (NOT 15) + assert len(redact_calls) == 3, ( + f"Expected 3 redaction calls (one per block), got {len(redact_calls)}" + ) diff --git a/translators/pdf_translator.py b/translators/pdf_translator.py index 71c7072..d3669d7 100644 --- a/translators/pdf_translator.py +++ b/translators/pdf_translator.py @@ -37,7 +37,19 @@ logger = get_logger(__name__) MIN_FONT_SIZE = 4.5 # Font size reduction factor when text overflows its bounding box -FONT_SHRINK_FACTOR = 0.87 +FONT_SHRINK_FACTOR = 0.93 # Track B3.5: was 0.87 (too aggressive, broke hierarchy) + +# Maximum vertical expansion factor (relative to original bbox height) +# Track B3.5: was 1.5x; bumped to 3x so longer translations can flow down +MAX_VERTICAL_EXPANSION = 3.0 + +# Maximum font shrink for headings (font >= HEADING_MIN_SIZE points) +# Track B3.5: never shrink a heading below 90% of its original size +HEADING_MIN_SIZE = 14.0 +HEADING_MIN_SCALE = 0.90 + +# Maximum font shrink for body text +BODY_MIN_SCALE = 0.75 # RTL language codes RTL_LANGUAGES = frozenset({"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"}) @@ -309,8 +321,14 @@ class PDFTranslator: for block in blocks: if block.get("translated"): - for sub_rect in block["sub_bboxes"]: - page.add_redact_annot(fitz.Rect(sub_rect), fill=(1, 1, 1)) + # Track B3.5: single redaction per block, not per sub-bbox. + # This reduces the number of redact annots (and the + # number of resulting "drawings" in the output PDF) + # from ~100 down to ~10 per page. + page.add_redact_annot( + fitz.Rect(block["bbox"]), + fill=(1, 1, 1), + ) page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE) @@ -525,15 +543,26 @@ class PDFTranslator: block: Dict, font_path: Optional[str], is_rtl: bool, - ) -> None: + ) -> bool: """Write translated text into the block's bounding box. - Priority: respect original font size as much as possible. - Strategy: - 1. Try original rect at original font size. - 2. Expand bbox to page margins (same font size). - 3. Expand bbox vertically downward (same font size). - 4. Only THEN shrink font as a last resort, with a floor of 70% original. + Track B3.5: Smart overflow handling. Returns True if the block + was rendered cleanly, False if it was skipped (format loss). + + Strategy (in order, first success wins): + 0. Try original rect at original font size. + 1. Expand bbox to page margins horizontally, original font size. + 2. Expand bbox vertically (up to MAX_VERTICAL_EXPANSION x height), + original font size. + 3. Shrink font ONCE (FONT_SHRINK_FACTOR = 0.93), with expanded bbox. + 4. Shrink font AGAIN (0.87 cumulative), with expanded bbox. + 5. Heading: stop at HEADING_MIN_SCALE (90%). Body: BODY_MIN_SCALE (75%). + 6. If still overflow: SKIP the block, emit format_loss metric, write + a small placeholder so the user knows something was lost. + + Key insight: prefer LOOSING a block over DEGRADING the entire page's + font hierarchy. A title that becomes unreadable 8.5pt is worse than + a missing TOC entry. """ import fitz @@ -544,70 +573,112 @@ class PDFTranslator: color = self._int_to_rgb(block["color"]) align = fitz.TEXT_ALIGN_RIGHT if is_rtl else fitz.TEXT_ALIGN_LEFT - fontname = None + # PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'. + # If a custom font file is available, use it via fontfile (fontname ignored). + fontname = "helv" fontfile = font_path - # Step 1: original rect, original size - size = target_size - rc = self._try_insert(page, original_rect, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return + # Determine if this is a heading (larger font size = more visual weight) + is_heading = target_size >= HEADING_MIN_SIZE + min_scale = HEADING_MIN_SCALE if is_heading else BODY_MIN_SCALE + min_size = max(target_size * min_scale, MIN_FONT_SIZE) - # Step 2: expand to page margins (horizontal) page_rect = page.rect margin = 18 - expanded_h = fitz.Rect( - max(original_rect.x0, page_rect.x0 + margin), - original_rect.y0, - min(original_rect.x1, page_rect.x1 - margin), - original_rect.y1, - ) - if expanded_h.width > original_rect.width: - rc = self._try_insert(page, expanded_h, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return - # Step 3: expand vertically (allow text to flow down) - max_expand_y = min(page_rect.y1 - margin - original_rect.y1, original_rect.height * 1.5) - expanded = fitz.Rect( + # Try to find a wider bbox that respects nearby blocks + # (don't overlap with the next text block on the page) + max_x1 = page_rect.x1 - margin + if not is_heading: + # For body text, allow horizontal expansion up to the page margin + expanded_h = fitz.Rect( + max(original_rect.x0, page_rect.x0 + margin), + original_rect.y0, + max_x1, + original_rect.y1, + ) + else: + # For headings, keep original width (preserves visual alignment) + expanded_h = original_rect + + # Try to find vertical room below this block + next_block_y = page_rect.y1 - margin + # (page.next_block_y is not exposed, so we use page bottom as ceiling) + max_expand_y = min( + next_block_y - original_rect.y1, + original_rect.height * MAX_VERTICAL_EXPANSION, + ) + expanded_v = fitz.Rect( expanded_h.x0, expanded_h.y0, expanded_h.x1, expanded_h.y1 + max_expand_y, ) - if expanded.height > expanded_h.height: - rc = self._try_insert(page, expanded, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return - # Step 4: shrink font — but never below 70% of original - min_size = max(target_size * 0.70, MIN_FONT_SIZE) - rect = expanded - for attempt in range(8): - size *= FONT_SHRINK_FACTOR - if size < min_size: - size = min_size - rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align) - break - rc = self._try_insert(page, rect, translated, size, fontname, fontfile, color, align) - if rc is not None and rc >= 0: - return + # Tier 0: original rect, original size + # insert_textbox returns >= 0 if text fit, < 0 if overflow (in points) + rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True - # Last resort - if rc is None or rc < 0: - try: - page.insert_textbox( - rect, - translated, - fontsize=min_size, - fontname=fontname or "helv", - fontfile=fontfile, - color=color, - align=align, - overlay=True, - ) - except Exception as e: - logger.warning("textbox_final_failed", error=str(e)) + # Tier 1: expanded horizontal, original size + if expanded_h.width > original_rect.width + 1: + rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 2: expanded vertical, original size + if expanded_v.height > original_rect.height + 1: + rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 3: shrink once + size_1 = max(target_size * FONT_SHRINK_FACTOR, min_size) + if size_1 < target_size - 0.1: # only try if shrink is meaningful + rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 4: shrink twice + size_2 = max(target_size * FONT_SHRINK_FACTOR * FONT_SHRINK_FACTOR, min_size) + if size_2 < size_1 - 0.1: + rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 5: hit the min size floor + rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align) + if rc is not None and rc >= 0: + return True + + # Tier 6: graceful failure — skip the block, log it + self._skip_block_with_marker(page, original_rect, translated, color) + _record_format_loss_metric("text_overflow") + logger.warning( + "pdf_block_skipped_overflow", + font_size=target_size, + text_preview=translated[:60], + ) + return False + + def _skip_block_with_marker(self, page, rect, original_text, color): + """Mark a skipped block with a visible placeholder so the user + knows something was lost. Uses a tiny font + grey color.""" + import fitz + placeholder = "[translation overflow]" + try: + page.insert_textbox( + rect, + placeholder, + fontsize=6.0, + fontname="helv", + color=(0.6, 0.6, 0.6), + align=fitz.TEXT_ALIGN_LEFT, + overlay=True, + ) + except Exception: + pass def _try_insert( self, page, rect, text, fontsize, fontname, fontfile, color, align @@ -624,7 +695,16 @@ class PDFTranslator: align=align, overlay=True, ) - except Exception: + except Exception as e: + # Demote this from warning to debug — insert_textbox is called + # 6+ times per block (one per tier), and these errors are common + # in normal operation (e.g. when the text really doesn't fit). + logger.debug( + "pdf_insert_textbox_error", + error=str(e)[:200], + error_type=type(e).__name__, + fontsize=fontsize, + ) return None @staticmethod