sepehr
d92bbf0fa6
Le defaut le plus sournois du pipeline etait silencieux : de l'arabe livre pour une cible persane (meme ecriture, mauvaise langue) passe inapercu et part chez le lecteur. Desormais chaque lot traduit est verifie par le detecteur d'ecritures, et chaque segment fautif est redemande une fois au moteur avec une consigne renforcee (nom de la langue + lettres specifiques, ex. persan پ چ ژ گ). La seconde tentative ne remplace la premiere que si elle passe le meme controle. - services/quality/script_detector.py : extraction d'un controle script_issue() reutilisable ; detect_arabic_variant signale des ormais un long texte en ecriture arabe sans aucune lettre specifique de la langue cible (arabe pur livre pour du persan/ourdou/pachto) - translators/segments.py : retry_wrong_script() + construction de la consigne renforcee, sans jamais faire echouer le travail - Word, Excel, PDF : branchement apres la memoire de traduction et les validations humaines ; le texte inchange (chiffres, noms propres) ne declenche jamais de retentative - 14 tests nouveaux (tests/test_translators/test_script_retry.py)
1859 lines
73 KiB
Python
1859 lines
73 KiB
Python
"""
|
||
PDF Document Translation Module — Layout-Preserving
|
||
|
||
Primary strategy (layout mode):
|
||
Use PyMuPDF (fitz) for direct, in-place text replacement on each page.
|
||
|
||
For each page:
|
||
1. Extract text blocks with positions, fonts, sizes, colors
|
||
2. Translate each block as a unit (preserving context within block)
|
||
3. Redact original text area
|
||
4. Write translated text at the same position, auto-adjusting font size
|
||
|
||
This preserves:
|
||
- Page structure, images, vector graphics, backgrounds
|
||
- Text positions within original bounding boxes
|
||
- Approximate font styling (size, color)
|
||
|
||
Fallback:
|
||
If PyMuPDF direct editing fails, falls back to the pdf2docx pipeline
|
||
(PDF → DOCX → WordTranslator → PDF via LibreOffice).
|
||
|
||
Text-only mode:
|
||
Extract text, translate, generate a clean formatted PDF via reportlab.
|
||
|
||
Scanned PDFs:
|
||
Image-only PDFs have no text layer to extract or rewrite. They are
|
||
detected up front (average extractable characters per page below
|
||
config.SCANNED_PDF_MIN_CHARS_PER_PAGE) and routed through the Mistral
|
||
OCR API (services/mistral_ocr.py) before translation; the output is a
|
||
clean re-typeset PDF (layout is not preserved — the source is images).
|
||
"""
|
||
|
||
import time
|
||
import shutil
|
||
import subprocess
|
||
import zlib
|
||
from pathlib import Path
|
||
from typing import Dict, Any, Optional, Callable, List
|
||
|
||
from core.logging import get_logger
|
||
from core.languages import is_rtl
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
# Minimum readable font size (points)
|
||
MIN_FONT_SIZE = 4.5
|
||
|
||
# Font size reduction factor when text overflows its bounding box
|
||
FONT_SHRINK_FACTOR = 0.93 # Track B3.5: was 0.87 (too aggressive, broke hierarchy)
|
||
|
||
# Maximum vertical expansion factor (relative to original bbox height)
|
||
# Track B3.5: was 1.5x; bumped to 3x so longer translations can flow down
|
||
# Track B3.6: bumped to 6x to handle long French titles that wrap to
|
||
# multiple lines (a 22pt title may need 4-5 lines in French).
|
||
MAX_VERTICAL_EXPANSION = 6.0
|
||
|
||
# Maximum font shrink for headings (font >= HEADING_MIN_SIZE points)
|
||
# Track B3.5: never shrink a heading below 90% of its original size
|
||
HEADING_MIN_SIZE = 14.0
|
||
HEADING_MIN_SCALE = 0.90
|
||
|
||
# Maximum font shrink for body text
|
||
BODY_MIN_SCALE = 0.75
|
||
|
||
# Warn at most once when the RTL shaping libraries are missing.
|
||
_RTL_SHAPING_MISSING_WARNED = False
|
||
|
||
|
||
def _shape_rtl(text: str) -> str:
|
||
"""Shape Arabic-script text for PDF engines that do no bidi of their own.
|
||
|
||
arabic-reshaper joins the letters into their contextual forms and
|
||
python-bidi reorders the string for visual (left-to-right) rendering.
|
||
Both libraries are optional on purpose: when they are missing or
|
||
fail, the text is returned unchanged with a warning — the
|
||
translation itself must always succeed, only the rendering degrades.
|
||
"""
|
||
global _RTL_SHAPING_MISSING_WARNED
|
||
if not text:
|
||
return text
|
||
try:
|
||
import arabic_reshaper
|
||
from bidi.algorithm import get_display
|
||
except Exception:
|
||
if not _RTL_SHAPING_MISSING_WARNED:
|
||
logger.warning(
|
||
"rtl_shaping_libs_missing",
|
||
hint="pip install arabic-reshaper python-bidi for correct RTL rendering",
|
||
)
|
||
_RTL_SHAPING_MISSING_WARNED = True
|
||
return text
|
||
try:
|
||
return get_display(arabic_reshaper.reshape(text))
|
||
except Exception as e:
|
||
logger.warning("rtl_shaping_failed", error=str(e)[:200])
|
||
return text
|
||
|
||
|
||
# Font path → reportlab font name, for RTL fonts registered in this
|
||
# process. A fixed name would make two consecutive translations with
|
||
# different fonts (e.g. Arabic then Hebrew) overwrite each other's
|
||
# registration.
|
||
_RTL_FONT_REGISTRATIONS: Dict[str, str] = {}
|
||
|
||
|
||
def _register_rtl_font(font_path: str) -> str:
|
||
"""Register a TTF under a stable per-path name, once per process.
|
||
|
||
Returns the reportlab font name, or "Helvetica" when registration
|
||
fails (warning logged — a font problem must never fail the job).
|
||
"""
|
||
registered = _RTL_FONT_REGISTRATIONS.get(font_path)
|
||
if registered:
|
||
return registered
|
||
try:
|
||
from reportlab.pdfbase import pdfmetrics
|
||
from reportlab.pdfbase.ttfonts import TTFont
|
||
|
||
digest = format(zlib.crc32(str(font_path).encode("utf-8")) & 0xFFFFFFFF, "08x")
|
||
name = f"WordlyRTL-{digest}"
|
||
pdfmetrics.registerFont(TTFont(name, font_path))
|
||
_RTL_FONT_REGISTRATIONS[font_path] = name
|
||
return name
|
||
except Exception as e:
|
||
logger.warning(
|
||
"pdf_rtl_font_register_failed", error=str(e)[:200], font=font_path
|
||
)
|
||
return "Helvetica"
|
||
|
||
|
||
def _record_format_loss_metric(element_type: str, count: int = 1) -> None:
|
||
"""Best-effort emission of a format-loss metric (never raises).
|
||
|
||
Used by the PDF translator to track hyperlinks/annotations lost
|
||
during in-place text replacement.
|
||
"""
|
||
if count <= 0:
|
||
return
|
||
try:
|
||
from middleware.metrics import record_format_loss
|
||
for _ in range(count):
|
||
record_format_loss(format="pdf", element_type=element_type)
|
||
except Exception:
|
||
# Metrics must never break a job
|
||
pass
|
||
|
||
|
||
def _libreoffice_available() -> bool:
|
||
"""Check if the LibreOffice CLI is available on this host.
|
||
|
||
Result is cached for 5 minutes to avoid spawning a subprocess
|
||
on every PDF translation.
|
||
"""
|
||
import time
|
||
now = time.time()
|
||
cached = _libreoffice_available._cache
|
||
if cached and (now - cached[0]) < 300:
|
||
return cached[1]
|
||
try:
|
||
result = subprocess.run(
|
||
["libreoffice", "--version"],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=5,
|
||
)
|
||
ok = result.returncode == 0
|
||
except (FileNotFoundError, subprocess.TimeoutExpired, Exception):
|
||
ok = False
|
||
_libreoffice_available._cache = (now, ok)
|
||
return ok
|
||
|
||
|
||
_libreoffice_available._cache = None # type: ignore[attr-defined]
|
||
|
||
|
||
# Script groups among the RTL languages — the preferred font depends on
|
||
# the writing system, not just the direction: a Hebrew target would
|
||
# render as tofu with an Arabic-only font.
|
||
_ARABIC_SCRIPT_LANGUAGES = frozenset(
|
||
{"ar", "fa", "ur", "ps", "ku", "sd", "ug", "ckb"}
|
||
)
|
||
_HEBREW_SCRIPT_LANGUAGES = frozenset({"he", "yi"})
|
||
|
||
|
||
def _rtl_script(code: str) -> str:
|
||
"""Script group of an RTL language code: "arabic", "hebrew" or ""
|
||
|
||
("" meaning: use the generic Unicode font list — e.g. Thaana "dv",
|
||
for which no dedicated font is searched).
|
||
"""
|
||
base = (code or "").strip().replace("_", "-").split("-")[0].lower()
|
||
if base in _ARABIC_SCRIPT_LANGUAGES:
|
||
return "arabic"
|
||
if base in _HEBREW_SCRIPT_LANGUAGES:
|
||
return "hebrew"
|
||
return ""
|
||
|
||
|
||
class PDFTranslator:
|
||
"""Translates PDF files with layout preservation using PyMuPDF."""
|
||
|
||
_FONT_SEARCH_PATHS = [
|
||
"/usr/share/fonts/opentype/noto/NotoSans-Regular.ttf",
|
||
"/usr/share/fonts/opentype/noto/NotoSans.ttc",
|
||
"/usr/share/fonts/truetype/noto/NotoSans-Regular.ttf",
|
||
"/usr/share/fonts/truetype/noto/NotoSans[Noto].ttf",
|
||
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
|
||
"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
|
||
"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
|
||
"/app/fonts/NotoSans-Regular.ttf",
|
||
# CJK-capable fonts (target languages zh/ja/ko render as tofu with
|
||
# a Latin-only font file)
|
||
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
||
"/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc",
|
||
"/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf",
|
||
"C:/Windows/Fonts/arial.ttf",
|
||
"C:/Windows/Fonts/msyh.ttc",
|
||
"C:/Windows/Fonts/simsun.ttc",
|
||
"C:/Windows/Fonts/msgothic.ttc",
|
||
"/System/Library/Fonts/Helvetica.ttc",
|
||
]
|
||
|
||
# Arabic-script-capable fonts (ar, fa, ur, ps, ku, sd, ug, ckb),
|
||
# searched FIRST for those targets. Debian's fonts-noto package
|
||
# ships Noto Naskh Arabic and Noto Sans Arabic under
|
||
# /usr/share/fonts/truetype/noto/; on Windows, Arial covers the
|
||
# whole Arabic script.
|
||
_RTL_ARABIC_FONT_SEARCH_PATHS = [
|
||
"/usr/share/fonts/truetype/noto/NotoNaskhArabic-Regular.ttf",
|
||
"/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf",
|
||
"/usr/share/fonts/opentype/noto/NotoNaskhArabic-Regular.ttf",
|
||
"/usr/share/fonts/opentype/noto/NotoSansArabic-Regular.ttf",
|
||
"/app/fonts/NotoNaskhArabic-Regular.ttf",
|
||
"/app/fonts/NotoSansArabic-Regular.ttf",
|
||
"C:/Windows/Fonts/arial.ttf",
|
||
]
|
||
|
||
# Hebrew-script-capable fonts (he, yi), searched FIRST for those
|
||
# targets. Same Debian package provides Noto Sans Hebrew; Arial
|
||
# covers Hebrew on Windows.
|
||
_RTL_HEBREW_FONT_SEARCH_PATHS = [
|
||
"/usr/share/fonts/truetype/noto/NotoSansHebrew-Regular.ttf",
|
||
"/usr/share/fonts/opentype/noto/NotoSansHebrew-Regular.ttf",
|
||
"/app/fonts/NotoSansHebrew-Regular.ttf",
|
||
"C:/Windows/Fonts/arial.ttf",
|
||
]
|
||
|
||
def __init__(self, provider=None):
|
||
self._provider = provider
|
||
self._font_path: Optional[str] = None
|
||
# script group ("arabic"/"hebrew"/"") → resolved font path, so a
|
||
# single instance can serve Arabic and Hebrew jobs in a row.
|
||
self._rtl_font_paths: Dict[str, str] = {}
|
||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||
self._custom_prompt: Optional[str] = None
|
||
# OCR overrides (admin settings); None → fall back to config.MISTRAL_*
|
||
self._ocr_api_key: Optional[str] = None
|
||
self._ocr_model: Optional[str] = None
|
||
self._ocr_timeout: Optional[int] = None
|
||
self._ocr_enabled: Optional[bool] = None
|
||
|
||
def set_provider(self, provider) -> None:
|
||
"""Set the translation provider."""
|
||
self._provider = provider
|
||
|
||
def set_custom_prompt(self, prompt: Optional[str]) -> None:
|
||
"""Set custom system prompt for LLM providers."""
|
||
self._custom_prompt = prompt
|
||
|
||
def set_ocr_config(
|
||
self,
|
||
api_key: Optional[str] = None,
|
||
model: Optional[str] = None,
|
||
timeout: Optional[int] = None,
|
||
enabled: Optional[bool] = None,
|
||
) -> None:
|
||
"""Configure the Mistral OCR call (admin settings > env defaults).
|
||
|
||
Only the values explicitly provided override the config defaults,
|
||
so the route can pass admin-configured values and let the rest fall
|
||
back to ``config.MISTRAL_*``.
|
||
"""
|
||
self._ocr_api_key = api_key
|
||
self._ocr_model = model
|
||
if timeout is not None:
|
||
self._ocr_timeout = timeout
|
||
if enabled is not None:
|
||
self._ocr_enabled = enabled
|
||
|
||
def set_segment_recorder(self, recorder) -> None:
|
||
"""Attach a segment recorder (review foundation)."""
|
||
self._segment_recorder = recorder
|
||
|
||
def set_segment_overrides(self, overrides) -> None:
|
||
"""Human-reviewed translations applied verbatim on rebuild."""
|
||
self._segment_overrides = overrides or {}
|
||
|
||
def get_recorded_segments(self):
|
||
recorder = getattr(self, "_segment_recorder", None)
|
||
return recorder.get_pairs() if recorder is not None else []
|
||
|
||
@staticmethod
|
||
def _first_existing_font(paths) -> Optional[str]:
|
||
for p in paths:
|
||
if Path(p).exists():
|
||
return p
|
||
return None
|
||
|
||
def _get_font_path(self, target_language: str = "") -> Optional[str]:
|
||
"""Resolve a Unicode-capable TTF/OTF font file.
|
||
|
||
RTL targets first search a font covering THEIR script — Noto
|
||
Naskh/Sans Arabic for the Arabic script (ar, fa, ur…), Noto Sans
|
||
Hebrew for Hebrew (he, yi); other RTL scripts (Thaana "dv") use
|
||
the generic Unicode list directly. A missing script font falls
|
||
back to the generic list with a warning — a missing font
|
||
degrades the rendering but never fails the job. Non-RTL targets
|
||
use the generic list only (unchanged behavior).
|
||
"""
|
||
if not is_rtl(target_language):
|
||
if self._font_path:
|
||
return self._font_path
|
||
found = self._first_existing_font(self._FONT_SEARCH_PATHS)
|
||
if not found:
|
||
logger.warning("no_unicode_font_found")
|
||
return None
|
||
self._font_path = found
|
||
return found
|
||
|
||
script = _rtl_script(target_language)
|
||
cached = self._rtl_font_paths.get(script)
|
||
if cached:
|
||
return cached
|
||
|
||
if script == "arabic":
|
||
preferred = self._RTL_ARABIC_FONT_SEARCH_PATHS
|
||
elif script == "hebrew":
|
||
preferred = self._RTL_HEBREW_FONT_SEARCH_PATHS
|
||
else:
|
||
preferred = []
|
||
|
||
found = self._first_existing_font(preferred + self._FONT_SEARCH_PATHS)
|
||
if found:
|
||
if preferred and found not in preferred:
|
||
logger.warning(
|
||
"rtl_script_font_not_found_current_font_used",
|
||
script=script,
|
||
font=found,
|
||
)
|
||
self._rtl_font_paths[script] = found
|
||
return found
|
||
logger.warning("no_unicode_font_found")
|
||
return None
|
||
|
||
def translate_file(
|
||
self,
|
||
input_path: Path,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str = "auto",
|
||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||
pdf_mode: str = "layout",
|
||
translate_images: bool = False,
|
||
**kwargs,
|
||
) -> Path:
|
||
input_path = Path(input_path)
|
||
output_path = Path(output_path)
|
||
self._validate_file(input_path)
|
||
|
||
# Scanned PDFs must be detected before either mode: both rely on an
|
||
# extractable text layer, which image-only pages don't have.
|
||
if self._is_scanned_pdf(input_path):
|
||
return self._translate_scanned_pdf(
|
||
input_path, output_path, target_language, source_language,
|
||
progress_callback,
|
||
)
|
||
|
||
if pdf_mode == "text_only":
|
||
return self._translate_text_only(
|
||
input_path, output_path, target_language, source_language, progress_callback
|
||
)
|
||
return self._translate_preserve_layout(
|
||
input_path, output_path, target_language, source_language, progress_callback,
|
||
translate_images=translate_images,
|
||
)
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# LAYOUT MODE — PyMuPDF in-place text replacement
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _translate_preserve_layout(
|
||
self,
|
||
input_path: Path,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str,
|
||
progress_callback,
|
||
translate_images: bool = False,
|
||
) -> Path:
|
||
"""Translate PDF preserving layout via PyMuPDF direct text replacement."""
|
||
start_time = time.time()
|
||
|
||
try:
|
||
import fitz
|
||
except ImportError:
|
||
logger.warning("pymupdf_missing_fallback_docx")
|
||
return self._translate_preserve_layout_fallback(
|
||
input_path, output_path, target_language, source_language, progress_callback,
|
||
translate_images=translate_images,
|
||
)
|
||
|
||
doc = fitz.open(str(input_path))
|
||
total_pages = len(doc)
|
||
|
||
if total_pages == 0:
|
||
doc.close()
|
||
raise RuntimeError("PDF has no pages.")
|
||
|
||
font_path = self._get_font_path(target_language)
|
||
logger.info(
|
||
"pdf_layout_start",
|
||
pages=total_pages,
|
||
file=input_path.name,
|
||
font=font_path or "built-in",
|
||
)
|
||
|
||
if progress_callback:
|
||
progress_callback({
|
||
"current": 1, "total": total_pages,
|
||
"phase": "extracting",
|
||
"paragraph": 1, "total_paragraphs": total_pages,
|
||
})
|
||
|
||
try:
|
||
result_path = self._process_pages_inplace(
|
||
doc, total_pages, output_path,
|
||
target_language, source_language,
|
||
font_path, progress_callback,
|
||
)
|
||
|
||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||
logger.info(
|
||
"pdf_layout_success",
|
||
pages=total_pages,
|
||
processing_time_ms=processing_time_ms,
|
||
output=str(result_path),
|
||
)
|
||
return result_path
|
||
|
||
except Exception as e:
|
||
doc.close()
|
||
logger.warning("inplace_failed_fallback", error=str(e))
|
||
return self._translate_preserve_layout_fallback(
|
||
input_path, output_path, target_language, source_language, progress_callback,
|
||
translate_images=translate_images,
|
||
)
|
||
|
||
def _process_pages_inplace(
|
||
self,
|
||
doc,
|
||
total_pages: int,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str,
|
||
font_path: Optional[str],
|
||
progress_callback,
|
||
) -> Path:
|
||
"""Core PyMuPDF in-place processing — one page at a time."""
|
||
import fitz
|
||
|
||
rtl_target = is_rtl(target_language)
|
||
total_blocks = 0
|
||
translated_blocks = 0
|
||
|
||
for page_num in range(total_pages):
|
||
page = doc[page_num]
|
||
raw_blocks = self._extract_text_blocks(page)
|
||
|
||
if not raw_blocks:
|
||
if progress_callback:
|
||
pct = int(30 + 65 * (page_num + 1) / total_pages)
|
||
progress_callback({
|
||
"current": page_num + 1, "total": total_pages,
|
||
"phase": f"Page {page_num + 1}/{total_pages} (no text)",
|
||
"paragraph": page_num + 1,
|
||
"total_paragraphs": total_pages,
|
||
"progress_override": pct,
|
||
})
|
||
continue
|
||
|
||
# Merge adjacent blocks that form a single paragraph
|
||
blocks = self._merge_adjacent_blocks(raw_blocks, page.rect)
|
||
total_blocks += len(blocks)
|
||
|
||
# Track B3.7: compute each block's "next block" y0 so the
|
||
# smart-fit logic never expands a block into its neighbour.
|
||
# Without this, a long translated block can flow into the
|
||
# block below it (e.g. "Pour la dernière version" overlapping
|
||
# "8. Résolution des problèmes").
|
||
self._populate_next_block_y(blocks, page.rect)
|
||
|
||
# Phase 1: translate all blocks on this page
|
||
for block in blocks:
|
||
original = block["text"]
|
||
if not original.strip():
|
||
continue
|
||
|
||
try:
|
||
translated = self._translate_single(
|
||
original, target_language, source_language
|
||
)
|
||
if translated and translated.strip():
|
||
if translated.strip() == original.strip():
|
||
# Unchanged (already in the target language, or
|
||
# the provider failed and returned the source
|
||
# text). Leave the block completely untouched —
|
||
# redacting and rewriting identical text would
|
||
# only degrade its typography (embedded fonts
|
||
# lost, everything redrawn in the substitute).
|
||
block["translated"] = None
|
||
logger.info(
|
||
"block_translation_unchanged",
|
||
page=page_num + 1,
|
||
text_preview=original[:60],
|
||
)
|
||
else:
|
||
block["translated"] = translated
|
||
translated_blocks += 1
|
||
else:
|
||
logger.warning(
|
||
"block_translation_empty",
|
||
page=page_num + 1,
|
||
text_preview=original[:60],
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"block_translation_failed",
|
||
page=page_num + 1,
|
||
error=str(e),
|
||
)
|
||
|
||
# Phase 2: redact original text areas
|
||
#
|
||
# CRITICAL (Track B3.6): For blocks that contain a vector
|
||
# drawing (e.g. a colored background box), we need to use a
|
||
# TRANSPARENT redaction (fill=None) so the original drawing
|
||
# isn't erased. For all other blocks, we use the standard
|
||
# white redaction to fully erase the text.
|
||
#
|
||
# Before redaction, we also capture the page's hyperlinks
|
||
# (PDF links are not stored in text blocks; they're attached
|
||
# to page annotations). After redaction, we re-insert the
|
||
# links so the translated page remains navigable.
|
||
#
|
||
# PDF_REDACT_IMAGE_NONE is critical: it tells PyMuPDF NOT to
|
||
# rasterize image content that intersects redaction areas.
|
||
# Without this flag, the redaction would replace the text
|
||
# AND the underlying image with a blank rectangle — wiping
|
||
# out any diagrams, charts, or photos in the same area.
|
||
page_links_before: list = []
|
||
if hasattr(page, "get_links"):
|
||
try:
|
||
page_links_before = list(page.get_links())
|
||
except Exception:
|
||
page_links_before = []
|
||
|
||
# Find the rectangles of all vector drawings on the page.
|
||
# If a block's bbox intersects a drawing rect, we use
|
||
# transparent redaction to preserve the drawing.
|
||
drawing_rects: list = []
|
||
if hasattr(page, "get_drawings"):
|
||
try:
|
||
for d in page.get_drawings():
|
||
r = d.get("rect")
|
||
if r is not None and d.get("fill") is not None:
|
||
drawing_rects.append(fitz.Rect(r))
|
||
except Exception:
|
||
pass
|
||
|
||
def _block_intersects_drawing(block_bbox):
|
||
"""True if the block's bbox intersects any drawing rect."""
|
||
if not drawing_rects:
|
||
return False
|
||
b = fitz.Rect(block_bbox)
|
||
for dr in drawing_rects:
|
||
if b.intersects(dr):
|
||
return True
|
||
return False
|
||
|
||
redacted_rects: list = []
|
||
for block in blocks:
|
||
if block.get("translated"):
|
||
# Track B3.5: single redaction per block, not per sub-bbox.
|
||
# Track B3.6: if the block contains a drawing, use
|
||
# transparent redaction (fill=None) so the drawing
|
||
# underneath isn't erased. Otherwise use white fill
|
||
# to fully remove the original text.
|
||
block_bbox = fitz.Rect(block["bbox"])
|
||
if _block_intersects_drawing(block_bbox):
|
||
# Transparent redaction: removes text but keeps
|
||
# the underlying drawing intact.
|
||
page.add_redact_annot(block_bbox, fill=None)
|
||
else:
|
||
page.add_redact_annot(block_bbox, fill=(1, 1, 1))
|
||
redacted_rects.append(block_bbox)
|
||
|
||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||
|
||
# Re-insert hyperlinks that were inside the redacted area.
|
||
# We use the original link geometry (it's unaffected by the
|
||
# redaction). URIs are preserved verbatim; only the visible
|
||
# text changes.
|
||
# Only links intersecting an actually-redacted block are
|
||
# re-inserted: when a page has no redaction at all (every block
|
||
# unchanged), the original annotations survive apply_redactions
|
||
# and re-inserting would DUPLICATE them.
|
||
if page_links_before and redacted_rects:
|
||
reinserted = 0
|
||
lost = 0
|
||
page_rect = page.rect
|
||
for link in page_links_before:
|
||
try:
|
||
from_rect = link.get("from")
|
||
if from_rect is None:
|
||
lost += 1
|
||
continue
|
||
# Clamp the rect to the page (in case redaction
|
||
# shrunk it)
|
||
if not page_rect.intersects(from_rect):
|
||
lost += 1
|
||
continue
|
||
# Skip links outside every redacted area — their
|
||
# original annotation is still alive on the page.
|
||
if not any(
|
||
from_rect.intersects(r) for r in redacted_rects
|
||
):
|
||
continue
|
||
# Build the link insertion kwargs based on kind
|
||
if link.get("uri"):
|
||
# External URI link
|
||
page.insert_link({
|
||
"kind": fitz.LINK_URI,
|
||
"from": from_rect,
|
||
"uri": link["uri"],
|
||
})
|
||
reinserted += 1
|
||
elif link.get("page") is not None:
|
||
# Internal goto link
|
||
page.insert_link({
|
||
"kind": fitz.LINK_GOTO,
|
||
"from": from_rect,
|
||
"page": link["page"],
|
||
"to": link.get("to", fitz.Point(0, 0)),
|
||
})
|
||
reinserted += 1
|
||
else:
|
||
lost += 1
|
||
except Exception:
|
||
lost += 1
|
||
if reinserted > 0:
|
||
logger.info(
|
||
"pdf_hyperlinks_reinserted",
|
||
page=page_num + 1,
|
||
reinserted=reinserted,
|
||
lost=lost,
|
||
)
|
||
if lost > 0:
|
||
_record_format_loss_metric("hyperlink", count=lost)
|
||
|
||
# Phase 3: write translated text
|
||
for block in blocks:
|
||
if block.get("translated"):
|
||
self._write_translated_block(
|
||
page, block, font_path, rtl_target
|
||
)
|
||
|
||
if progress_callback:
|
||
pct = int(30 + 65 * (page_num + 1) / total_pages)
|
||
progress_callback({
|
||
"current": page_num + 1, "total": total_pages,
|
||
"phase": f"Translating page {page_num + 1}/{total_pages}",
|
||
"paragraph": page_num + 1,
|
||
"total_paragraphs": total_pages,
|
||
"progress_override": pct,
|
||
})
|
||
|
||
logger.info(
|
||
"pdf_blocks_processed",
|
||
total_blocks=total_blocks,
|
||
translated_blocks=translated_blocks,
|
||
)
|
||
|
||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||
doc.save(str(output_path), garbage=4, deflate=True)
|
||
doc.close()
|
||
return output_path
|
||
|
||
def _extract_text_blocks(self, page) -> List[Dict]:
|
||
"""Extract text blocks with position, font, and color information.
|
||
|
||
Track B3.9: when a PDF "block" contains multiple LINES at the
|
||
same y (typical of table rows where each cell is a separate line
|
||
in the block), split it into one sub-block per line. This
|
||
preserves the column structure so a translated table row stays
|
||
a table row instead of collapsing into a single left-aligned
|
||
column.
|
||
|
||
Track B3.10: when a block is COVERED by a drawing (code block
|
||
with a background rect, callout box, etc.), split it into
|
||
per-line blocks and mark them as ``no_merge``. Without this,
|
||
the merge logic collapses 5 code lines into one block, and the
|
||
smart-fit rewrites them all at the block's bbox, which
|
||
destroys the line-by-line layout and the surrounding
|
||
background drawing's apparent extent.
|
||
|
||
The detection is: if a block has >= 2 lines whose y0 is within
|
||
``SAME_ROW_Y_TOLERANCE`` of each other, treat it as a
|
||
multi-column row and emit one sub-block per line. Blocks whose
|
||
lines are vertically stacked (multi-line paragraphs) keep the
|
||
old behavior, UNLESS they are covered by a drawing — in which
|
||
case we split per-line and set ``no_merge`` to preserve the
|
||
code-block / callout layout.
|
||
"""
|
||
import fitz
|
||
|
||
blocks = []
|
||
data = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
|
||
|
||
SAME_ROW_Y_TOLERANCE = 3.0 # points
|
||
|
||
# B3.10: pre-compute the set of page drawings with a fill
|
||
# (colored rects, callout boxes, etc.) so we can detect
|
||
# "code block" or "callout" patterns and avoid merging their
|
||
# constituent lines.
|
||
drawing_rects: list = []
|
||
if hasattr(page, "get_drawings"):
|
||
try:
|
||
for d in page.get_drawings():
|
||
r = d.get("rect")
|
||
if r is not None and d.get("fill") is not None:
|
||
drawing_rects.append(fitz.Rect(r))
|
||
except Exception:
|
||
pass
|
||
|
||
def _block_covered_by_drawing(block_bbox_tuple) -> bool:
|
||
"""True if the block's bbox is covered by a filled drawing.
|
||
|
||
We use "covered" loosely — a substantial intersection
|
||
(>= 50% of the block's area) qualifies. This avoids
|
||
false positives from drawings that merely touch a corner
|
||
of the block.
|
||
"""
|
||
if not drawing_rects:
|
||
return False
|
||
b = fitz.Rect(block_bbox_tuple)
|
||
block_area = b.width * b.height
|
||
if block_area <= 0:
|
||
return False
|
||
for dr in drawing_rects:
|
||
if not b.intersects(dr):
|
||
continue
|
||
# Compute intersection area
|
||
inter = b & dr
|
||
inter_area = inter.width * inter.height
|
||
if inter_area / block_area >= 0.5:
|
||
return True
|
||
return False
|
||
|
||
for block in data.get("blocks", []):
|
||
if block.get("type") != 0:
|
||
continue
|
||
|
||
lines = block.get("lines", [])
|
||
if not lines:
|
||
continue
|
||
|
||
# B3.9: detect table-row pattern. A row is multiple lines
|
||
# at the same y (different x). Multi-line paragraphs have
|
||
# lines at different y (same x).
|
||
line_y0s = [line["bbox"][1] for line in lines]
|
||
same_y = all(
|
||
abs(y - line_y0s[0]) <= SAME_ROW_Y_TOLERANCE
|
||
for y in line_y0s
|
||
)
|
||
has_horizontal_layout = (
|
||
same_y and len(lines) > 1
|
||
and any(
|
||
abs(lines[i]["bbox"][0] - lines[0]["bbox"][0]) > 5
|
||
for i in range(1, len(lines))
|
||
)
|
||
)
|
||
|
||
# B3.10: detect code-block / callout pattern — a block
|
||
# whose lines are covered by a colored background drawing.
|
||
# Note: each code line is its own PDF block, so we check
|
||
# the block bbox (which is the line's bbox) against the
|
||
# drawing, not "len(lines) > 1".
|
||
covered_by_drawing = (
|
||
not has_horizontal_layout
|
||
and _block_covered_by_drawing(block["bbox"])
|
||
)
|
||
|
||
if has_horizontal_layout or covered_by_drawing:
|
||
# Multi-column row OR code block: emit one block per line.
|
||
for line in lines:
|
||
span_parts = []
|
||
spans_info = []
|
||
for span in line.get("spans", []):
|
||
text = span.get("text", "")
|
||
if text:
|
||
span_parts.append(text)
|
||
spans_info.append({
|
||
"size": span.get("size", 12),
|
||
"font": span.get("font", "Helvetica"),
|
||
"color": span.get("color", 0),
|
||
"flags": span.get("flags", 0),
|
||
"origin": span.get("origin", (0, 0)),
|
||
})
|
||
if not span_parts:
|
||
continue
|
||
line_text = "".join(span_parts).strip()
|
||
if not line_text:
|
||
continue
|
||
avg_size = (
|
||
sum(s["size"] for s in spans_info) / len(spans_info)
|
||
if spans_info else 12.0
|
||
)
|
||
first_color = spans_info[0]["color"] if spans_info else 0
|
||
is_bold = any(s["flags"] & 16 for s in spans_info)
|
||
is_italic = any(s["flags"] & 2 for s in spans_info)
|
||
blocks.append({
|
||
"bbox": tuple(line["bbox"]),
|
||
"text": line_text,
|
||
"font_size": round(avg_size, 1),
|
||
"color": first_color,
|
||
"is_bold": is_bold,
|
||
"is_italic": is_italic,
|
||
"line_count": 1,
|
||
"translated": None,
|
||
"sub_bboxes": [tuple(line["bbox"])],
|
||
"_is_table_cell": has_horizontal_layout,
|
||
"_no_merge": covered_by_drawing,
|
||
})
|
||
continue
|
||
|
||
# Standard multi-line paragraph: join lines with \n.
|
||
line_parts = []
|
||
spans_info = []
|
||
|
||
for line in lines:
|
||
span_parts = []
|
||
for span in line.get("spans", []):
|
||
text = span.get("text", "")
|
||
if text:
|
||
span_parts.append(text)
|
||
spans_info.append({
|
||
"size": span.get("size", 12),
|
||
"font": span.get("font", "Helvetica"),
|
||
"color": span.get("color", 0),
|
||
"flags": span.get("flags", 0),
|
||
"origin": span.get("origin", (0, 0)),
|
||
})
|
||
if span_parts:
|
||
line_parts.append("".join(span_parts))
|
||
|
||
full_text = "\n".join(line_parts).strip()
|
||
if not full_text:
|
||
continue
|
||
|
||
avg_size = (
|
||
sum(s["size"] for s in spans_info) / len(spans_info)
|
||
if spans_info
|
||
else 12.0
|
||
)
|
||
first_color = spans_info[0]["color"] if spans_info else 0
|
||
is_bold = any(s["flags"] & 16 for s in spans_info)
|
||
is_italic = any(s["flags"] & 2 for s in spans_info)
|
||
|
||
blocks.append({
|
||
"bbox": tuple(block["bbox"]),
|
||
"text": full_text,
|
||
"font_size": round(avg_size, 1),
|
||
"color": first_color,
|
||
"is_bold": is_bold,
|
||
"is_italic": is_italic,
|
||
"line_count": len(line_parts),
|
||
"translated": None,
|
||
"sub_bboxes": [tuple(block["bbox"])],
|
||
})
|
||
|
||
return blocks
|
||
|
||
def _merge_adjacent_blocks(
|
||
self, blocks: List[Dict], page_rect
|
||
) -> List[Dict]:
|
||
"""Merge consecutive text blocks that form a single paragraph.
|
||
|
||
Blocks are merged when they:
|
||
- Have the same (or very close) font size
|
||
- Are vertically adjacent (gap < 1.5× line height)
|
||
- Have the same x-origin (left-aligned) or same width
|
||
This produces larger bounding boxes for better translation context
|
||
and prevents excessive font-size reduction for multi-line paragraphs.
|
||
"""
|
||
if len(blocks) <= 1:
|
||
return blocks
|
||
|
||
merged = []
|
||
current = dict(blocks[0])
|
||
|
||
for next_block in blocks[1:]:
|
||
should_merge = self._should_merge_blocks(current, next_block)
|
||
if should_merge:
|
||
# Merge: combine text and expand bounding box
|
||
current["text"] += "\n" + next_block["text"]
|
||
current["line_count"] += next_block["line_count"]
|
||
# Expand bbox to cover both
|
||
x0 = min(current["bbox"][0], next_block["bbox"][0])
|
||
y0 = min(current["bbox"][1], next_block["bbox"][1])
|
||
x1 = max(current["bbox"][2], next_block["bbox"][2])
|
||
y1 = max(current["bbox"][3], next_block["bbox"][3])
|
||
current["bbox"] = (x0, y0, x1, y1)
|
||
current["sub_bboxes"].extend(next_block["sub_bboxes"])
|
||
else:
|
||
merged.append(current)
|
||
current = dict(next_block)
|
||
|
||
merged.append(current)
|
||
return merged
|
||
|
||
def _should_merge_blocks(self, a: Dict, b: Dict) -> bool:
|
||
"""Check if two blocks should be merged into one paragraph."""
|
||
a_bbox = a["bbox"]
|
||
b_bbox = b["bbox"]
|
||
|
||
# Track B3.10: never merge blocks marked as no_merge.
|
||
# These are lines inside a code block, callout box, or any
|
||
# other structure that needs to preserve its line-by-line
|
||
# layout (e.g. each line of a curl command must stay at its
|
||
# own y position to remain aligned with the background
|
||
# drawing that covers them).
|
||
if a.get("_no_merge") or b.get("_no_merge"):
|
||
return False
|
||
|
||
# Table cells: consecutive rows of the same column satisfy every
|
||
# geometric merge condition (same x0, similar width, small positive
|
||
# gap) but are UNRELATED values — merging them joins two cells into
|
||
# one paragraph spanning both rows and breaks the table structure.
|
||
if a.get("_is_table_cell") or b.get("_is_table_cell"):
|
||
return False
|
||
|
||
# Must have similar font size (within 20%)
|
||
if abs(a["font_size"] - b["font_size"]) > max(a["font_size"], b["font_size"]) * 0.2:
|
||
return False
|
||
|
||
# Block b must start soon after block a ends vertically
|
||
vertical_gap = b_bbox[1] - a_bbox[3]
|
||
line_height = a["font_size"] * 1.4
|
||
if vertical_gap < 0 or vertical_gap > line_height * 1.5:
|
||
return False
|
||
|
||
# Similar horizontal position (within 15pt)
|
||
if abs(a_bbox[0] - b_bbox[0]) > 15:
|
||
return False
|
||
|
||
# Don't merge if widths are very different (likely different columns)
|
||
a_width = a_bbox[2] - a_bbox[0]
|
||
b_width = b_bbox[2] - b_bbox[0]
|
||
if a_width > 0 and abs(b_width - a_width) / a_width > 0.5:
|
||
return False
|
||
|
||
return True
|
||
|
||
def _populate_next_block_y(self, blocks: List[Dict], page_rect) -> None:
|
||
"""For each block, set ``block['_next_block_y']`` to the y0 of the
|
||
nearest block below it in the SAME COLUMN (or the page bottom
|
||
margin if none).
|
||
|
||
Track B3.7: this lets the smart-fit logic use the next block as
|
||
a ceiling for vertical expansion, so a long translated block
|
||
never overlaps its neighbour.
|
||
|
||
Track B3.8: COLUMN-AWARE. PDFs with multi-column layouts
|
||
(e.g. journal articles, brochures) would otherwise get
|
||
the wrong "next block" — sorting globally by y0 would map
|
||
a left-column block to a right-column block as its "next".
|
||
We group blocks into columns by x0 proximity (15pt tolerance),
|
||
then sort each column by y0. Blocks that don't fit any
|
||
column (e.g. full-width centered headers) are treated as
|
||
their own single-block column.
|
||
"""
|
||
margin = 18
|
||
page_bottom = page_rect.y1 - margin
|
||
COLUMN_TOLERANCE = 15.0 # points — blocks within this x0 are "same column"
|
||
|
||
if not blocks:
|
||
return
|
||
|
||
# Group blocks into columns by x0 proximity. We iterate in
|
||
# x0-sorted order and assign each block to the first column
|
||
# whose reference x0 is within tolerance, or start a new column.
|
||
# This gives us columns ordered left-to-right.
|
||
columns: List[List[Dict]] = []
|
||
for block in sorted(blocks, key=lambda b: b["bbox"][0]):
|
||
assigned = False
|
||
for column in columns:
|
||
col_ref_x0 = column[0]["bbox"][0]
|
||
if abs(block["bbox"][0] - col_ref_x0) <= COLUMN_TOLERANCE:
|
||
column.append(block)
|
||
assigned = True
|
||
break
|
||
if not assigned:
|
||
columns.append([block])
|
||
|
||
# For each column, set next_block_y by y-order within that column.
|
||
# A block whose expanded bbox would touch the next column-mate's
|
||
# y0 will be capped to that y0 - 2 (small visual gap).
|
||
for column in columns:
|
||
column.sort(key=lambda b: b["bbox"][1])
|
||
for i, block in enumerate(column):
|
||
if i + 1 < len(column):
|
||
block["_next_block_y"] = column[i + 1]["bbox"][1] - 2
|
||
else:
|
||
block["_next_block_y"] = page_bottom
|
||
|
||
def _write_translated_block(
|
||
self,
|
||
page,
|
||
block: Dict,
|
||
font_path: Optional[str],
|
||
rtl_target: bool,
|
||
) -> bool:
|
||
"""Write translated text into the block's bounding box.
|
||
|
||
Track B3.5: Smart overflow handling. Returns True if the block
|
||
was rendered cleanly, False if it was skipped (format loss).
|
||
|
||
Strategy (in order, first success wins):
|
||
0. Try original rect at original font size.
|
||
1. Expand bbox to page margins horizontally, original font size.
|
||
2. Expand bbox vertically (up to MAX_VERTICAL_EXPANSION x height),
|
||
original font size.
|
||
3. Shrink font ONCE (FONT_SHRINK_FACTOR = 0.93), with expanded bbox.
|
||
4. Shrink font AGAIN (0.87 cumulative), with expanded bbox.
|
||
5. Heading: stop at HEADING_MIN_SCALE (90%). Body: BODY_MIN_SCALE (75%).
|
||
6. If still overflow: SKIP the block, emit format_loss metric, write
|
||
a small placeholder so the user knows something was lost.
|
||
|
||
Key insight: prefer LOOSING a block over DEGRADING the entire page's
|
||
font hierarchy. A title that becomes unreadable 8.5pt is worse than
|
||
a missing TOC entry.
|
||
"""
|
||
import fitz
|
||
|
||
original_rect = fitz.Rect(block["bbox"])
|
||
translated = block["translated"]
|
||
if rtl_target:
|
||
# Shape Arabic-script text (contextual letter forms + visual
|
||
# order) before insertion — insert_textbox does no bidi
|
||
# processing of its own. Missing/failed libs degrade to the
|
||
# raw text (warning logged inside _shape_rtl).
|
||
translated = _shape_rtl(translated)
|
||
target_size = block["font_size"]
|
||
|
||
color = self._int_to_rgb(block["color"])
|
||
align = fitz.TEXT_ALIGN_RIGHT if is_rtl else fitz.TEXT_ALIGN_LEFT
|
||
|
||
# PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'.
|
||
# If a custom font file is available, use it via fontfile (fontname ignored).
|
||
# Base-14 font variant honouring the block's bold/italic flags —
|
||
# previously every block (headings included) rendered in regular.
|
||
# When a custom Unicode fontfile is used we keep it: we have no
|
||
# bold/italic variant of that file, and glyph coverage matters more
|
||
# than weight.
|
||
if font_path:
|
||
fontname = "helv"
|
||
fontfile = font_path
|
||
else:
|
||
fontname = (
|
||
"hebi" if (block.get("is_bold") and block.get("is_italic"))
|
||
else "hebo" if block.get("is_bold")
|
||
else "heit" if block.get("is_italic")
|
||
else "helv"
|
||
)
|
||
fontfile = None
|
||
|
||
# Determine if this is a heading (larger font size = more visual weight)
|
||
is_heading = target_size >= HEADING_MIN_SIZE
|
||
min_scale = HEADING_MIN_SCALE if is_heading else BODY_MIN_SCALE
|
||
min_size = max(target_size * min_scale, MIN_FONT_SIZE)
|
||
|
||
page_rect = page.rect
|
||
margin = 18
|
||
|
||
# Track B3.6: try a wider bbox that respects the page margin.
|
||
# For headings, also allow horizontal expansion because long
|
||
# translated titles often don't fit in the original width.
|
||
max_x1 = page_rect.x1 - margin
|
||
expanded_h = fitz.Rect(
|
||
max(original_rect.x0, page_rect.x0 + margin),
|
||
original_rect.y0,
|
||
max_x1,
|
||
original_rect.y1,
|
||
)
|
||
|
||
# Track B3.7: cap vertical expansion at the next block's y0
|
||
# (not the page bottom), so a long translated block cannot
|
||
# overflow into its neighbour.
|
||
# Track B3.8: clamp to >= 0 to avoid an invalid (inverted) Rect
|
||
# when next_block_y sits above the current block (rare edge
|
||
# case in extracted blocks where bbox ordering is unusual).
|
||
next_block_y = block.get("_next_block_y", page_rect.y1 - margin)
|
||
max_expand_y = max(
|
||
0.0,
|
||
min(
|
||
next_block_y - original_rect.y1,
|
||
original_rect.height * MAX_VERTICAL_EXPANSION,
|
||
),
|
||
)
|
||
expanded_v = fitz.Rect(
|
||
expanded_h.x0,
|
||
expanded_h.y0,
|
||
expanded_h.x1,
|
||
expanded_h.y1 + max_expand_y,
|
||
)
|
||
|
||
# Tier 0: original rect, original size
|
||
# insert_textbox returns >= 0 if text fit, < 0 if overflow (in points)
|
||
rc = self._try_insert(page, original_rect, translated, target_size, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 1: expanded horizontal, original size
|
||
if expanded_h.width > original_rect.width + 1:
|
||
rc = self._try_insert(page, expanded_h, translated, target_size, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 2: expanded vertical, original size
|
||
if expanded_v.height > original_rect.height + 1:
|
||
rc = self._try_insert(page, expanded_v, translated, target_size, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 3: shrink once
|
||
size_1 = max(target_size * FONT_SHRINK_FACTOR, min_size)
|
||
if size_1 < target_size - 0.1: # only try if shrink is meaningful
|
||
rc = self._try_insert(page, expanded_v, translated, size_1, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 4: shrink twice
|
||
size_2 = max(target_size * FONT_SHRINK_FACTOR * FONT_SHRINK_FACTOR, min_size)
|
||
if size_2 < size_1 - 0.1:
|
||
rc = self._try_insert(page, expanded_v, translated, size_2, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 5: hit the min size floor
|
||
rc = self._try_insert(page, expanded_v, translated, min_size, fontname, fontfile, color, align)
|
||
if rc is not None and rc >= 0:
|
||
return True
|
||
|
||
# Tier 6: graceful failure — skip the block, log it
|
||
self._skip_block_with_marker(page, original_rect, translated, color)
|
||
_record_format_loss_metric("text_overflow")
|
||
logger.warning(
|
||
"pdf_block_skipped_overflow",
|
||
font_size=target_size,
|
||
text_preview=translated[:60],
|
||
)
|
||
return False
|
||
|
||
def _skip_block_with_marker(self, page, rect, original_text, color):
|
||
"""Mark a skipped block with a visible placeholder so the user
|
||
knows something was lost. Uses a tiny font + grey color."""
|
||
import fitz
|
||
placeholder = "[translation overflow]"
|
||
try:
|
||
page.insert_textbox(
|
||
rect,
|
||
placeholder,
|
||
fontsize=6.0,
|
||
fontname="helv",
|
||
color=(0.6, 0.6, 0.6),
|
||
align=fitz.TEXT_ALIGN_LEFT,
|
||
overlay=True,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _try_insert(
|
||
self, page, rect, text, fontsize, fontname, fontfile, color, align
|
||
):
|
||
"""Attempt insert_textbox, returns rc or None on error."""
|
||
try:
|
||
return page.insert_textbox(
|
||
rect,
|
||
text,
|
||
fontsize=fontsize,
|
||
fontname=fontname,
|
||
fontfile=fontfile,
|
||
color=color,
|
||
align=align,
|
||
overlay=True,
|
||
)
|
||
except Exception as e:
|
||
# Demote this from warning to debug — insert_textbox is called
|
||
# 6+ times per block (one per tier), and these errors are common
|
||
# in normal operation (e.g. when the text really doesn't fit).
|
||
logger.debug(
|
||
"pdf_insert_textbox_error",
|
||
error=str(e)[:200],
|
||
error_type=type(e).__name__,
|
||
fontsize=fontsize,
|
||
)
|
||
return None
|
||
|
||
@staticmethod
|
||
def _int_to_rgb(color_int: int) -> tuple:
|
||
"""Convert integer color (0xRRGGBB) to (r, g, b) float tuple."""
|
||
r = ((color_int >> 16) & 0xFF) / 255.0
|
||
g = ((color_int >> 8) & 0xFF) / 255.0
|
||
b = (color_int & 0xFF) / 255.0
|
||
return (r, g, b)
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# FALLBACK — pdf2docx → WordTranslator → LibreOffice
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _translate_preserve_layout_fallback(
|
||
self,
|
||
input_path: Path,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str,
|
||
progress_callback,
|
||
translate_images: bool = False,
|
||
) -> Path:
|
||
"""Fallback: PDF → DOCX (pdf2docx) → WordTranslator → PDF (LibreOffice)."""
|
||
start_time = time.time()
|
||
|
||
try:
|
||
if progress_callback:
|
||
progress_callback({
|
||
"current": 1, "total": 3,
|
||
"phase": "converting",
|
||
"paragraph": 1, "total_paragraphs": 3,
|
||
})
|
||
|
||
docx_path = self._convert_pdf_to_docx(input_path)
|
||
|
||
if progress_callback:
|
||
progress_callback({
|
||
"current": 2, "total": 3,
|
||
"phase": "translating",
|
||
"paragraph": 2, "total_paragraphs": 3,
|
||
})
|
||
|
||
from translators.word_translator import WordTranslator
|
||
|
||
translated_docx = output_path.with_suffix(".docx")
|
||
wt = WordTranslator(provider=self._provider)
|
||
wt.translate_file(
|
||
docx_path, translated_docx,
|
||
target_language, source_language,
|
||
progress_callback=None,
|
||
translate_images=translate_images,
|
||
)
|
||
# Propagate the inner Word stats so the route's sanity gate
|
||
# (attempted/changed) works on the fallback path too.
|
||
for key, value in wt.get_translation_stats().items():
|
||
self._translation_stats[key] = (
|
||
self._translation_stats.get(key, 0) + value
|
||
)
|
||
|
||
if progress_callback:
|
||
progress_callback({
|
||
"current": 3, "total": 3,
|
||
"phase": "converting_back",
|
||
"paragraph": 3, "total_paragraphs": 3,
|
||
})
|
||
|
||
final_path = self._convert_docx_to_pdf(translated_docx, output_path)
|
||
|
||
for tmp in [docx_path, translated_docx]:
|
||
if tmp.exists() and tmp != final_path:
|
||
try:
|
||
tmp.unlink()
|
||
except Exception:
|
||
pass
|
||
|
||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||
logger.info(
|
||
"pdf_layout_fallback_success",
|
||
file_name=input_path.name,
|
||
processing_time_ms=processing_time_ms,
|
||
output=str(final_path),
|
||
)
|
||
return final_path
|
||
|
||
except Exception as e:
|
||
logger.error("pdf_layout_fallback_error", file=str(input_path), error=str(e))
|
||
raise
|
||
|
||
def _convert_pdf_to_docx(self, pdf_path: Path) -> Path:
|
||
"""Convert PDF to DOCX using pdf2docx."""
|
||
try:
|
||
from pdf2docx import Converter
|
||
except ImportError:
|
||
raise RuntimeError("pdf2docx is not installed")
|
||
|
||
docx_path = pdf_path.with_suffix(".docx")
|
||
cv = Converter(str(pdf_path))
|
||
try:
|
||
cv.convert(str(docx_path))
|
||
finally:
|
||
cv.close()
|
||
|
||
if not docx_path.exists() or docx_path.stat().st_size == 0:
|
||
raise RuntimeError("PDF conversion produced empty output")
|
||
|
||
from docx import Document
|
||
doc = Document(str(docx_path))
|
||
total_text = "".join(p.text for p in doc.paragraphs).strip()
|
||
if not total_text:
|
||
raise RuntimeError("PDF appears to be scanned or contains only images")
|
||
|
||
logger.info("pdf_converted_to_docx", pages=len(doc.paragraphs))
|
||
return docx_path
|
||
|
||
def _convert_docx_to_pdf(self, docx_path: Path, target_pdf: Path) -> Path:
|
||
"""Convert DOCX → PDF using LibreOffice headless."""
|
||
if not _libreoffice_available():
|
||
# Cache miss → known-missing path
|
||
logger.warning(
|
||
"libreoffice_not_found",
|
||
hint="Install libreoffice-core on the host for DOCX→PDF fallback",
|
||
)
|
||
return None # type: ignore[return-value]
|
||
try:
|
||
result = subprocess.run(
|
||
[
|
||
"libreoffice", "--headless", "--convert-to", "pdf",
|
||
"--outdir", str(target_pdf.parent),
|
||
str(docx_path),
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=120,
|
||
)
|
||
expected_pdf = docx_path.with_suffix(".pdf")
|
||
if expected_pdf.exists() and expected_pdf.stat().st_size > 0:
|
||
if expected_pdf != target_pdf:
|
||
shutil.move(str(expected_pdf), str(target_pdf))
|
||
logger.info("docx_to_pdf_success")
|
||
return target_pdf
|
||
logger.warning("docx_to_pdf_no_output", stderr=result.stderr[:200])
|
||
except FileNotFoundError:
|
||
logger.warning("libreoffice_not_found_runtime")
|
||
except subprocess.TimeoutExpired:
|
||
logger.warning("libreoffice_timeout")
|
||
except Exception as e:
|
||
logger.warning("docx_to_pdf_failed", error=str(e)[:200])
|
||
|
||
docx_output = target_pdf.with_suffix(".docx")
|
||
if docx_path != docx_output and docx_path.exists():
|
||
shutil.move(str(docx_path), str(docx_output))
|
||
return docx_output
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# MODE: text_only — extract text, translate, clean PDF output
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _translate_text_only(
|
||
self,
|
||
input_path: Path,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str,
|
||
progress_callback,
|
||
) -> Path:
|
||
"""Extract text from PDF, translate, output as a clean formatted PDF."""
|
||
import fitz
|
||
|
||
start_time = time.time()
|
||
doc = fitz.open(str(input_path))
|
||
total_pages = len(doc)
|
||
|
||
if total_pages == 0:
|
||
doc.close()
|
||
raise RuntimeError("PDF has no pages.")
|
||
|
||
logger.info("pdf_text_only_start", pages=total_pages, file=input_path.name)
|
||
|
||
pages_text = []
|
||
for page_num in range(total_pages):
|
||
page = doc[page_num]
|
||
text = page.get_text("text").strip()
|
||
pages_text.append(text)
|
||
doc.close()
|
||
|
||
translated_pages = self._translate_page_texts(
|
||
pages_text, target_language, source_language, progress_callback
|
||
)
|
||
|
||
final_path = output_path.with_suffix(".pdf")
|
||
self._generate_clean_pdf(translated_pages, final_path, target_language)
|
||
|
||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||
logger.info(
|
||
"pdf_text_only_success",
|
||
file_name=input_path.name,
|
||
pages=total_pages,
|
||
processing_time_ms=processing_time_ms,
|
||
)
|
||
|
||
return final_path
|
||
|
||
def _translate_page_texts(
|
||
self,
|
||
pages_text: List[str],
|
||
target_language: str,
|
||
source_language: str,
|
||
progress_callback,
|
||
) -> List[str]:
|
||
"""Translate per-page texts, keeping order; empty pages pass through."""
|
||
non_empty_indices = [i for i, t in enumerate(pages_text) if t]
|
||
|
||
if progress_callback:
|
||
progress_callback({
|
||
"current": 1, "total": 3,
|
||
"phase": "translating",
|
||
"paragraph": 1, "total_paragraphs": 3,
|
||
})
|
||
|
||
translated_pages = list(pages_text)
|
||
total_pages = len(pages_text)
|
||
|
||
for seq, page_idx in enumerate(non_empty_indices):
|
||
text = pages_text[page_idx]
|
||
if not text.strip():
|
||
continue
|
||
try:
|
||
translated = self._translate_single(text, target_language, source_language)
|
||
if translated and translated.strip():
|
||
translated_pages[page_idx] = translated
|
||
else:
|
||
logger.warning("page_translation_empty", page=page_idx + 1)
|
||
except Exception as e:
|
||
logger.warning("page_translation_failed", page=page_idx + 1, error=str(e))
|
||
|
||
if progress_callback:
|
||
pct = int(30 + 60 * (seq + 1) / len(non_empty_indices))
|
||
progress_callback({
|
||
"current": seq + 1,
|
||
"total": len(non_empty_indices),
|
||
"phase": f"Translating page {page_idx + 1}/{total_pages}",
|
||
"paragraph": seq + 1,
|
||
"total_paragraphs": len(non_empty_indices),
|
||
"progress_override": pct,
|
||
})
|
||
|
||
return translated_pages
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# SCANNED PDFs — Mistral OCR
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _is_scanned_pdf(self, input_path: Path) -> bool:
|
||
"""True when the PDF is image-only (no usable text layer).
|
||
|
||
A page counts as a scan page when it has almost no extractable
|
||
text AND is mostly covered by raster images. The document is
|
||
considered scanned when it contains scan pages and no page with a
|
||
real text layer — this keeps sparse-but-textual PDFs (a bare
|
||
title page, a single label) on the normal layout pipeline.
|
||
"""
|
||
from config import config
|
||
|
||
try:
|
||
import fitz
|
||
except ImportError:
|
||
return False
|
||
|
||
try:
|
||
with fitz.open(str(input_path)) as doc:
|
||
if len(doc) == 0:
|
||
return False
|
||
has_text_page = False
|
||
has_scan_page = False
|
||
for page in doc:
|
||
if len(page.get_text("text").strip()) >= config.SCANNED_PDF_MIN_CHARS_PER_PAGE:
|
||
has_text_page = True
|
||
continue
|
||
# Text-poor page: a scan page only when raster images
|
||
# cover most of its area (a bare title page has none).
|
||
page_area = abs(page.rect)
|
||
img_area = 0.0
|
||
for img in page.get_images(full=True):
|
||
for rect in page.get_image_rects(img[0]):
|
||
img_area += abs(rect & page.rect)
|
||
if page_area and img_area / page_area >= 0.5:
|
||
has_scan_page = True
|
||
except Exception as e:
|
||
logger.warning("scanned_pdf_detection_failed", error=str(e))
|
||
return False
|
||
|
||
scanned = has_scan_page and not has_text_page
|
||
if scanned:
|
||
logger.info(
|
||
"scanned_pdf_detected",
|
||
file=input_path.name,
|
||
)
|
||
return scanned
|
||
|
||
def _translate_scanned_pdf(
|
||
self,
|
||
input_path: Path,
|
||
output_path: Path,
|
||
target_language: str,
|
||
source_language: str,
|
||
progress_callback,
|
||
) -> Path:
|
||
"""OCR (Mistral) → translate → clean re-typeset PDF.
|
||
|
||
The source pages are images, so the original layout cannot be
|
||
rewritten in place; the output carries the recovered text in a
|
||
clean document instead.
|
||
"""
|
||
from config import config
|
||
from services.mistral_ocr import MistralOCRClient, MistralOCRError
|
||
|
||
# Resolution order: admin settings (via set_ocr_config) > env/config.
|
||
api_key = (self._ocr_api_key or "").strip() or config.MISTRAL_API_KEY
|
||
model = (self._ocr_model or "").strip() or config.MISTRAL_OCR_MODEL
|
||
timeout = self._ocr_timeout or config.MISTRAL_OCR_TIMEOUT
|
||
ocr_enabled = (
|
||
config.MISTRAL_OCR_ENABLED
|
||
if self._ocr_enabled is None
|
||
else self._ocr_enabled
|
||
)
|
||
|
||
if not ocr_enabled or not api_key:
|
||
raise RuntimeError(
|
||
"PDF scanné détecté (pages image sans couche texte). "
|
||
"La traduction des PDF scannés nécessite l'OCR Mistral : "
|
||
"configurez MISTRAL_API_KEY (ou fournissez un PDF avec du texte sélectionnable)."
|
||
)
|
||
|
||
start_time = time.time()
|
||
client = MistralOCRClient(
|
||
api_key=api_key,
|
||
model=model,
|
||
timeout=timeout,
|
||
)
|
||
|
||
pages_markdown = client.extract_pdf_text(
|
||
input_path, progress_callback=progress_callback
|
||
)
|
||
pages_text = [self._markdown_to_text(md) for md in pages_markdown]
|
||
|
||
translated_pages = self._translate_page_texts(
|
||
pages_text, target_language, source_language, progress_callback
|
||
)
|
||
|
||
final_path = output_path.with_suffix(".pdf")
|
||
self._generate_clean_pdf(translated_pages, final_path, target_language)
|
||
|
||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||
logger.info(
|
||
"pdf_scanned_success",
|
||
file_name=input_path.name,
|
||
pages=len(pages_text),
|
||
processing_time_ms=processing_time_ms,
|
||
)
|
||
return final_path
|
||
|
||
@staticmethod
|
||
def _markdown_to_text(markdown: str) -> str:
|
||
"""Flatten OCR markdown to plain text (drop images/links/markup)."""
|
||
import re
|
||
|
||
if not markdown:
|
||
return ""
|
||
text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", markdown) # images
|
||
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) # links → label
|
||
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) # headings
|
||
text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) # bullets
|
||
# Markdown table rows → plain line of cells
|
||
text = re.sub(r"^\s*\|", "", text, flags=re.MULTILINE)
|
||
text = text.replace("|", " ")
|
||
text = re.sub(r"^\s*[-:| ]+\s*$", "", text, flags=re.MULTILINE) # rules
|
||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||
return text.strip()
|
||
|
||
def _generate_clean_pdf(
|
||
self, pages_text: List[str], output_path: Path, target_language: str = "en"
|
||
) -> None:
|
||
"""Generate a clean, well-formatted PDF from translated page texts."""
|
||
from reportlab.lib.pagesizes import A4
|
||
from reportlab.lib.styles import ParagraphStyle
|
||
from reportlab.lib.units import mm
|
||
from reportlab.lib.enums import TA_LEFT, TA_JUSTIFY, TA_RIGHT
|
||
from reportlab.lib import colors
|
||
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak
|
||
from reportlab.lib.styles import getSampleStyleSheet
|
||
|
||
rtl_target = is_rtl(target_language)
|
||
alignment = TA_RIGHT if rtl_target else TA_JUSTIFY
|
||
|
||
# RTL targets need a TTF font covering the target script (the
|
||
# built-in Helvetica has no Arabic/Hebrew glyphs). Registration
|
||
# failures degrade to Helvetica with a warning — never a failed
|
||
# job.
|
||
font_name = "Helvetica"
|
||
if rtl_target:
|
||
rtl_font = self._get_font_path(target_language)
|
||
if rtl_font:
|
||
font_name = _register_rtl_font(rtl_font)
|
||
else:
|
||
logger.warning("pdf_rtl_font_missing_helvetica_fallback")
|
||
|
||
styles = getSampleStyleSheet()
|
||
|
||
pdf_doc = SimpleDocTemplate(
|
||
str(output_path),
|
||
pagesize=A4,
|
||
leftMargin=25 * mm,
|
||
rightMargin=25 * mm,
|
||
topMargin=25 * mm,
|
||
bottomMargin=25 * mm,
|
||
)
|
||
|
||
body_style = ParagraphStyle(
|
||
"BodyText_Custom",
|
||
parent=styles["Normal"],
|
||
fontSize=11,
|
||
leading=16,
|
||
spaceAfter=6,
|
||
alignment=alignment,
|
||
fontName=font_name,
|
||
wordWrap="RTL" if rtl_target else None,
|
||
textColor=colors.HexColor("#1a1a1a"),
|
||
)
|
||
|
||
page_number_style = ParagraphStyle(
|
||
"PageNumber",
|
||
parent=styles["Normal"],
|
||
fontSize=9,
|
||
textColor=colors.HexColor("#999999"),
|
||
alignment=TA_LEFT,
|
||
)
|
||
|
||
elements = []
|
||
for i, page_text in enumerate(pages_text):
|
||
if not page_text.strip():
|
||
continue
|
||
|
||
if len(pages_text) > 1:
|
||
elements.append(Paragraph(f"— Page {i + 1} —", page_number_style))
|
||
elements.append(Spacer(1, 8))
|
||
|
||
for para_text in page_text.split("\n"):
|
||
para_text = para_text.strip()
|
||
if not para_text:
|
||
elements.append(Spacer(1, 4))
|
||
continue
|
||
|
||
if rtl_target:
|
||
# Shape Arabic-script text before rendering —
|
||
# reportlab does no bidi processing of its own.
|
||
para_text = _shape_rtl(para_text)
|
||
|
||
safe = (
|
||
para_text
|
||
.replace("&", "&")
|
||
.replace("<", "<")
|
||
.replace(">", ">")
|
||
)
|
||
|
||
try:
|
||
elements.append(Paragraph(safe, body_style))
|
||
except Exception:
|
||
elements.append(
|
||
Paragraph(
|
||
para_text.encode("ascii", "replace").decode(),
|
||
body_style,
|
||
)
|
||
)
|
||
|
||
if i < len(pages_text) - 1:
|
||
elements.append(PageBreak())
|
||
|
||
if not elements:
|
||
raise RuntimeError("No text content to generate PDF")
|
||
|
||
pdf_doc.build(elements)
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# Shared helpers
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _translate_with_provider(
|
||
self, texts: List[str], target_language: str, source_language: str,
|
||
extra_prompt: Optional[str] = None,
|
||
) -> List[str]:
|
||
"""Translate using the TranslationProvider interface (handles old & new styles)."""
|
||
from services.providers.base import TranslationProvider as NewTranslationProvider
|
||
|
||
is_new_style = False
|
||
if isinstance(self._provider, NewTranslationProvider):
|
||
is_new_style = True
|
||
elif hasattr(self._provider, "__class__") and self._provider.__class__.__name__ in (
|
||
"MockTranslationProvider",
|
||
"Mock",
|
||
"MagicMock",
|
||
):
|
||
is_new_style = True
|
||
|
||
if is_new_style:
|
||
from services.providers.schemas import TranslationRequest
|
||
custom_prompt = getattr(self, "_custom_prompt", None)
|
||
if extra_prompt:
|
||
custom_prompt = (
|
||
f"{custom_prompt}\n\n{extra_prompt}" if custom_prompt else extra_prompt
|
||
)
|
||
metadata = {"custom_prompt": custom_prompt} if custom_prompt else None
|
||
|
||
requests = [
|
||
TranslationRequest(
|
||
text=t,
|
||
target_language=target_language,
|
||
source_language=source_language,
|
||
metadata=metadata,
|
||
)
|
||
for t in texts
|
||
]
|
||
responses = self._provider.translate_batch(requests)
|
||
translated = [resp.translated_text for resp in responses]
|
||
else:
|
||
translated = self._provider.translate_batch(texts, target_language, source_language)
|
||
|
||
# Fallback: keep original text for any empty/failed result
|
||
return [
|
||
t if (t and t.strip()) else orig
|
||
for t, orig in zip(translated, texts)
|
||
]
|
||
|
||
def _translate_single(
|
||
self, text: str, target_language: str, source_language: str
|
||
) -> str:
|
||
"""Translate a single text string.
|
||
|
||
Also feeds the job-level attempted/changed stats so the route can
|
||
detect a total provider failure (changed == 0) on PDFs too.
|
||
Reviewer overrides (approved/edited segments) short-circuit the
|
||
provider entirely — verbatim reviewed text, zero drift.
|
||
"""
|
||
overrides = getattr(self, "_segment_overrides", None)
|
||
if overrides:
|
||
override = overrides.get((text or "").strip())
|
||
if override and override.strip() and override.strip() != (text or "").strip():
|
||
recorder = getattr(self, "_segment_recorder", None)
|
||
if recorder is not None:
|
||
recorder.record_pair(text, override)
|
||
return override
|
||
|
||
if text and text.strip():
|
||
self._translation_stats["attempted"] += 1
|
||
if self._provider is not None:
|
||
try:
|
||
results = self._translate_with_provider([text], target_language, source_language)
|
||
if results and results[0].strip():
|
||
result = results[0]
|
||
if result.strip() != text.strip():
|
||
self._translation_stats["changed"] += 1
|
||
recorder = getattr(self, "_segment_recorder", None)
|
||
if recorder is not None:
|
||
recorder.record_pair(text, result)
|
||
return result
|
||
except Exception as e:
|
||
logger.warning("provider_single_failed", error=str(e))
|
||
|
||
from services.translation_service import translation_service
|
||
try:
|
||
result = translation_service.translate_text(text, target_language, source_language)
|
||
if result and result.strip() and result.strip() != text.strip():
|
||
self._translation_stats["changed"] += 1
|
||
recorder = getattr(self, "_segment_recorder", None)
|
||
if recorder is not None:
|
||
recorder.record_pair(text, result)
|
||
return result
|
||
except Exception as e:
|
||
logger.warning("legacy_single_failed", error=str(e))
|
||
return text
|
||
|
||
def _translate_batch(
|
||
self, texts: List[str], target_language: str, source_language: str
|
||
) -> List[str]:
|
||
"""Translate a batch of texts."""
|
||
non_empty = [t for t in texts if t and t.strip()]
|
||
self._translation_stats["attempted"] += len(non_empty)
|
||
|
||
translated = None
|
||
if self._provider is not None:
|
||
try:
|
||
translated = self._translate_with_provider(texts, target_language, source_language)
|
||
except Exception as e:
|
||
logger.warning("provider_translate_failed", error=str(e))
|
||
|
||
if translated is None:
|
||
from services.translation_service import translation_service
|
||
try:
|
||
translated = translation_service.translate_batch(texts, target_language, source_language)
|
||
except Exception as e:
|
||
logger.warning("legacy_translate_failed", error=str(e))
|
||
translated = texts
|
||
|
||
# Wrong-script guard: re-ask once (with a reinforced instruction)
|
||
# for translations delivered in the wrong alphabet — e.g. Arabic
|
||
# delivered for a Persian target.
|
||
from translators.segments import retry_wrong_script
|
||
|
||
def _retry_translate(retry_texts, hint):
|
||
if self._provider is not None:
|
||
return self._translate_with_provider(
|
||
retry_texts, target_language, source_language, extra_prompt=hint
|
||
)
|
||
from services.translation_service import translation_service
|
||
return translation_service.translate_batch(
|
||
retry_texts, target_language, source_language
|
||
)
|
||
|
||
translated, _script_fixed = retry_wrong_script(
|
||
texts, translated, target_language, _retry_translate
|
||
)
|
||
|
||
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
|
||
self._translation_stats["changed"] += changed
|
||
|
||
return translated
|
||
|
||
def get_translation_stats(self) -> dict:
|
||
return dict(self._translation_stats)
|
||
|
||
def _validate_file(self, file_path: Path) -> None:
|
||
if not file_path.exists():
|
||
raise FileNotFoundError(f"File not found: {file_path.name}")
|
||
if file_path.suffix.lower() != ".pdf":
|
||
raise ValueError(f"Expected .pdf file, got {file_path.suffix}")
|
||
with open(file_path, "rb") as f:
|
||
header = f.read(5)
|
||
if header[:4] != b"%PDF":
|
||
raise ValueError("File does not appear to be a valid PDF.")
|
||
|
||
|
||
pdf_translator = PDFTranslator()
|