feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s
Translation quality & format preservation: - Word: merge adjacent same-format runs into one unit (sentence-level coherence like inline-tag handling); translate comments/balloons; dedupe textbox collection (was translated twice); RTL no longer overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs) - PPTX: chart translations now actually reach the output file (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP); CJK typeface hints (a:ea) - Excel: sheet renames no longer break references — rewrite cell formulas (3D/quoted), defined names, data validations, cond. formats - PDF: bold/italic honored (hebo/heit/hebi); table cells never merge; unchanged blocks left untouched (typography preserved, fixes duplicate hyperlinks); attempted/changed stats + route gate now cover PDF; CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings) Features: - formality param (formal/informal) + automatic regional-variant prompts - output_mode=bilingual docx (source above translation) - per-user translation memory on Redis (falls back to LRU), context-hashed - QA report + 0-100 confidence score in job status; L0 on by default - OpenAI-compatible providers: whole chunk in ONE numbered-JSON request (~15x fewer calls) with per-item fallback; base prompt always present (custom prompt no longer replaces translation instructions) Infra & marketing alignment: - plan-based engine gating + vision gating (closes paid-engine leak); /providers/available filtered per plan; 107 languages exposed - zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash) - admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin (TestClient breakage); Prometheus test fixture fixed - marketing docs aligned with code (PDF+OCR, retention, engines, pricing) - security: .env.ionos/.env.production/provider_settings.json removed Tests: 1173 passed / 0 failed (6 network tests deselected: free Google endpoint temporarily blocked from this machine)
This commit is contained in:
90
translators/bilingual.py
Normal file
90
translators/bilingual.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Bilingual output: interleave source paragraphs with their translation.
|
||||
|
||||
Given the ORIGINAL document and the TRANSLATED document (same structure —
|
||||
the pipeline only rewrites run texts), produce a copy of the translated
|
||||
document where each translated body paragraph is preceded by its source
|
||||
paragraph in gray italic. Tables, headers and footers keep the translated
|
||||
version only (duplicating them would double the layout).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from docx import Document
|
||||
from docx.text.paragraph import Paragraph
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Pt, RGBColor
|
||||
|
||||
from core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def make_bilingual_docx(
|
||||
source_path: Path, translated_path: Path, output_path: Path
|
||||
) -> Optional[Path]:
|
||||
"""Create a bilingual .docx (source paragraph above its translation).
|
||||
|
||||
Returns the output path, or None when the pairing failed (structure
|
||||
mismatch) — callers fall back to the translated-only file.
|
||||
"""
|
||||
source_path = Path(source_path)
|
||||
translated_path = Path(translated_path)
|
||||
output_path = Path(output_path)
|
||||
|
||||
try:
|
||||
src = Document(str(source_path))
|
||||
tr = Document(str(translated_path))
|
||||
except Exception as e:
|
||||
logger.warning("bilingual_open_failed", error=str(e))
|
||||
return None
|
||||
|
||||
src_children = list(src.element.body)
|
||||
tr_children = list(tr.element.body)
|
||||
|
||||
# The pipeline preserves structure exactly; a drift beyond a small
|
||||
# tolerance means pairing by index is unsafe → bail out.
|
||||
if abs(len(src_children) - len(tr_children)) > 0:
|
||||
logger.warning(
|
||||
"bilingual_structure_mismatch",
|
||||
source_elements=len(src_children),
|
||||
translated_elements=len(tr_children),
|
||||
)
|
||||
if len(src_children) != len(tr_children):
|
||||
return None
|
||||
|
||||
inserted = 0
|
||||
for src_el, tr_el in zip(src_children, tr_children):
|
||||
if tr_el.tag != qn("w:p") or src_el.tag != qn("w:p"):
|
||||
continue
|
||||
src_text = "".join(
|
||||
t.text or "" for t in src_el.iter(qn("w:t"))
|
||||
).strip()
|
||||
if not src_text:
|
||||
continue
|
||||
|
||||
# Insert a NEW paragraph directly above the translated one, inside
|
||||
# the translated document (keeps styles/sections untouched).
|
||||
new_p = OxmlElement("w:p")
|
||||
tr_el.addprevious(new_p)
|
||||
para = Paragraph(new_p, tr)
|
||||
run = para.add_run(src_text)
|
||||
run.font.size = Pt(9)
|
||||
run.font.italic = True
|
||||
run.font.color.rgb = RGBColor(0x80, 0x80, 0x80)
|
||||
inserted += 1
|
||||
|
||||
if inserted == 0:
|
||||
logger.info("bilingual_nothing_inserted")
|
||||
return None
|
||||
|
||||
try:
|
||||
tr.save(str(output_path))
|
||||
except Exception as e:
|
||||
logger.warning("bilingual_save_failed", error=str(e))
|
||||
return None
|
||||
|
||||
logger.info("bilingual_docx_created", paragraphs=inserted)
|
||||
return output_path
|
||||
@@ -107,6 +107,7 @@ class ExcelTranslator:
|
||||
self._provider = provider
|
||||
self.formula_pattern = re.compile(r"=.*")
|
||||
self._custom_prompt: Optional[str] = None
|
||||
self._tm_scope = None # set via set_tm_scope (per-user translation memory)
|
||||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||||
|
||||
def set_provider(self, provider: TranslationProvider) -> None:
|
||||
@@ -116,6 +117,14 @@ class ExcelTranslator:
|
||||
def set_custom_prompt(self, prompt: Optional[str]) -> None:
|
||||
"""Set custom system prompt for LLM providers."""
|
||||
self._custom_prompt = prompt
|
||||
def set_tm_scope(self, user_id, prompt=None) -> None:
|
||||
"""Enable the per-user translation memory for this job."""
|
||||
from services.translation_tm import TMScope
|
||||
|
||||
self._tm_scope = TMScope.from_prompt(
|
||||
user_id, prompt or getattr(self, "_custom_prompt", None)
|
||||
)
|
||||
|
||||
|
||||
def translate_file(
|
||||
self,
|
||||
@@ -318,6 +327,13 @@ class ExcelTranslator:
|
||||
new_name=new_name,
|
||||
)
|
||||
|
||||
# openpyxl does NOT rewrite references on rename: cell
|
||||
# formulas, defined names and chart refs pointing at the old
|
||||
# sheet would break (#REF!/#NAME?). Charts are handled later
|
||||
# via ZIP re-injection; fix cells + defined names here.
|
||||
if sheet_name_mapping:
|
||||
self._rewrite_sheet_refs_in_workbook(workbook, sheet_name_mapping)
|
||||
|
||||
if translate_images:
|
||||
_log_info("excel_image_translation_start", sheets=len(workbook.sheetnames))
|
||||
for sheet_name in workbook.sheetnames:
|
||||
@@ -427,12 +443,30 @@ class ExcelTranslator:
|
||||
non_empty = [t for t in texts if t and t.strip()]
|
||||
self._translation_stats["attempted"] += len(non_empty)
|
||||
|
||||
from services.translation_tm import translate_with_tm
|
||||
|
||||
provider_name = (
|
||||
self._provider.get_name() if hasattr(self._provider, "get_name")
|
||||
else type(self._provider).__name__
|
||||
) if self._provider is not None else "legacy"
|
||||
|
||||
if self._provider is not None:
|
||||
translated = self._translate_with_provider(
|
||||
texts, target_language, source_language
|
||||
)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_provider(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
else:
|
||||
translated = self._translate_with_legacy(texts, target_language, source_language)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_legacy(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
|
||||
# Translation memory: reuse this user's previous translations
|
||||
# (identical context/prompt) before hitting the provider.
|
||||
translated = translate_with_tm(
|
||||
texts, target_language, source_language,
|
||||
provider_name, getattr(self, "_tm_scope", None), _do_translate,
|
||||
)
|
||||
|
||||
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
|
||||
self._translation_stats["changed"] += changed
|
||||
@@ -931,6 +965,140 @@ class ExcelTranslator:
|
||||
rewritten += 1
|
||||
return rewritten
|
||||
|
||||
@staticmethod
|
||||
def _rewrite_sheet_refs_in_formula(
|
||||
formula: str, sheet_name_mapping: Dict[str, str]
|
||||
) -> str:
|
||||
"""Rewrite every sheet reference inside a formula string.
|
||||
|
||||
Handles quoted ('Ventes 2026'!), unquoted (Ventes!) and 3D refs
|
||||
(Sheet1!A1:Sheet2!B2 — each end is rewritten independently).
|
||||
Longest names are replaced first so a name that is a prefix of
|
||||
another is not corrupted.
|
||||
"""
|
||||
if not formula or not sheet_name_mapping or "!" not in formula:
|
||||
return formula
|
||||
|
||||
result = formula
|
||||
# longest first to avoid partial-name collisions
|
||||
for old in sorted(sheet_name_mapping, key=len, reverse=True):
|
||||
new = sheet_name_mapping[old]
|
||||
if new == old:
|
||||
continue
|
||||
new_quoted = ExcelTranslator._quote_sheet_name_for_ref(new)
|
||||
# Quoted form: inner apostrophes are escaped as ''
|
||||
escaped = old.replace("'", "''")
|
||||
result = re.sub(
|
||||
rf"'{re.escape(escaped)}'!",
|
||||
new_quoted + "!",
|
||||
result,
|
||||
)
|
||||
# Unquoted form: only when the old name needs no quotes; guard
|
||||
# against matching the tail of a longer name.
|
||||
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.]*", old):
|
||||
result = re.sub(
|
||||
rf"(?<![A-Za-z0-9_.']){re.escape(old)}!",
|
||||
new_quoted + "!",
|
||||
result,
|
||||
)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _rewrite_sheet_refs_in_workbook(
|
||||
cls, workbook, sheet_name_mapping: Dict[str, str]
|
||||
) -> None:
|
||||
"""After a sheet rename, fix every reference that still points at the
|
||||
old names: cell formulas, defined names, data validations and
|
||||
conditional-formatting rules. openpyxl does none of this on rename.
|
||||
|
||||
Best-effort: individual failures are logged and skipped — a broken
|
||||
rename must never lose the whole file.
|
||||
"""
|
||||
if not sheet_name_mapping:
|
||||
return
|
||||
|
||||
cells_fixed = names_fixed = validations_fixed = cf_fixed = 0
|
||||
|
||||
try:
|
||||
for worksheet in workbook.worksheets:
|
||||
# 1) Cell formulas
|
||||
for row in worksheet.iter_rows():
|
||||
for cell in row:
|
||||
value = cell.value
|
||||
if isinstance(value, str) and value.startswith("=") and "!" in value:
|
||||
updated = cls._rewrite_sheet_refs_in_formula(
|
||||
value, sheet_name_mapping
|
||||
)
|
||||
if updated != value:
|
||||
cell.value = updated
|
||||
cells_fixed += 1
|
||||
|
||||
# 2) Data validations (list sources / custom formulas)
|
||||
try:
|
||||
for dv in worksheet.data_validations.dataValidation:
|
||||
for attr in ("formula1", "formula2"):
|
||||
fx = getattr(dv, attr, None)
|
||||
if isinstance(fx, str) and "!" in fx:
|
||||
updated = cls._rewrite_sheet_refs_in_formula(
|
||||
fx, sheet_name_mapping
|
||||
)
|
||||
if updated != fx:
|
||||
setattr(dv, attr, updated)
|
||||
validations_fixed += 1
|
||||
except Exception as e:
|
||||
_log_warning(
|
||||
"excel_sheet_refs_validations_failed",
|
||||
sheet=worksheet.title,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# 3) Conditional formatting rules
|
||||
try:
|
||||
for cf in worksheet.conditional_formatting:
|
||||
for rule in cf.rules:
|
||||
formulas = getattr(rule, "formula", None) or []
|
||||
for idx, fx in enumerate(formulas):
|
||||
if isinstance(fx, str) and "!" in fx:
|
||||
updated = cls._rewrite_sheet_refs_in_formula(
|
||||
fx, sheet_name_mapping
|
||||
)
|
||||
if updated != fx:
|
||||
formulas[idx] = updated
|
||||
cf_fixed += 1
|
||||
except Exception as e:
|
||||
_log_warning(
|
||||
"excel_sheet_refs_condfmt_failed",
|
||||
sheet=worksheet.title,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# 4) Workbook-level defined names
|
||||
try:
|
||||
for name in list(workbook.defined_names.values()):
|
||||
attr_text = getattr(name, "attr_text", None)
|
||||
if isinstance(attr_text, str) and "!" in attr_text:
|
||||
updated = cls._rewrite_sheet_refs_in_formula(
|
||||
attr_text, sheet_name_mapping
|
||||
)
|
||||
if updated != attr_text:
|
||||
name.attr_text = updated
|
||||
names_fixed += 1
|
||||
except Exception as e:
|
||||
_log_warning("excel_sheet_refs_names_failed", error=str(e))
|
||||
|
||||
except Exception as e:
|
||||
_log_error("excel_sheet_refs_rewrite_failed", error=str(e))
|
||||
return
|
||||
|
||||
if cells_fixed or names_fixed or validations_fixed or cf_fixed:
|
||||
_log_info(
|
||||
"excel_sheet_refs_rewritten",
|
||||
cells=cells_fixed,
|
||||
defined_names=names_fixed,
|
||||
validations=validations_fixed,
|
||||
conditional_formats=cf_fixed,
|
||||
)
|
||||
|
||||
def _translate_images(self, worksheet: Worksheet, target_language: str) -> None:
|
||||
"""
|
||||
Translate text in images using vision model.
|
||||
|
||||
@@ -21,6 +21,13 @@ Fallback:
|
||||
|
||||
Text-only mode:
|
||||
Extract text, translate, generate a clean formatted PDF via reportlab.
|
||||
|
||||
Scanned PDFs:
|
||||
Image-only PDFs have no text layer to extract or rewrite. They are
|
||||
detected up front (average extractable characters per page below
|
||||
config.SCANNED_PDF_MIN_CHARS_PER_PAGE) and routed through the Mistral
|
||||
OCR API (services/mistral_ocr.py) before translation; the output is a
|
||||
clean re-typeset PDF (layout is not preserved — the source is images).
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -114,8 +121,15 @@ class PDFTranslator:
|
||||
"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
|
||||
"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
|
||||
"/app/fonts/NotoSans-Regular.ttf",
|
||||
# CJK-capable fonts (target languages zh/ja/ko render as tofu with
|
||||
# a Latin-only font file)
|
||||
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
||||
"/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc",
|
||||
"/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf",
|
||||
"C:/Windows/Fonts/arial.ttf",
|
||||
"C:/Windows/Fonts/msyh.ttc",
|
||||
"C:/Windows/Fonts/simsun.ttc",
|
||||
"C:/Windows/Fonts/msgothic.ttc",
|
||||
"/System/Library/Fonts/Helvetica.ttc",
|
||||
]
|
||||
|
||||
@@ -124,6 +138,11 @@ class PDFTranslator:
|
||||
self._font_path: Optional[str] = None
|
||||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||||
self._custom_prompt: Optional[str] = None
|
||||
# OCR overrides (admin settings); None → fall back to config.MISTRAL_*
|
||||
self._ocr_api_key: Optional[str] = None
|
||||
self._ocr_model: Optional[str] = None
|
||||
self._ocr_timeout: Optional[int] = None
|
||||
self._ocr_enabled: Optional[bool] = None
|
||||
|
||||
def set_provider(self, provider) -> None:
|
||||
"""Set the translation provider."""
|
||||
@@ -133,6 +152,26 @@ class PDFTranslator:
|
||||
"""Set custom system prompt for LLM providers."""
|
||||
self._custom_prompt = prompt
|
||||
|
||||
def set_ocr_config(
|
||||
self,
|
||||
api_key: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
enabled: Optional[bool] = None,
|
||||
) -> None:
|
||||
"""Configure the Mistral OCR call (admin settings > env defaults).
|
||||
|
||||
Only the values explicitly provided override the config defaults,
|
||||
so the route can pass admin-configured values and let the rest fall
|
||||
back to ``config.MISTRAL_*``.
|
||||
"""
|
||||
self._ocr_api_key = api_key
|
||||
self._ocr_model = model
|
||||
if timeout is not None:
|
||||
self._ocr_timeout = timeout
|
||||
if enabled is not None:
|
||||
self._ocr_enabled = enabled
|
||||
|
||||
def _get_font_path(self) -> Optional[str]:
|
||||
"""Resolve a Unicode-capable TTF/OTF font file."""
|
||||
if self._font_path is not None:
|
||||
@@ -159,6 +198,14 @@ class PDFTranslator:
|
||||
output_path = Path(output_path)
|
||||
self._validate_file(input_path)
|
||||
|
||||
# Scanned PDFs must be detected before either mode: both rely on an
|
||||
# extractable text layer, which image-only pages don't have.
|
||||
if self._is_scanned_pdf(input_path):
|
||||
return self._translate_scanned_pdf(
|
||||
input_path, output_path, target_language, source_language,
|
||||
progress_callback,
|
||||
)
|
||||
|
||||
if pdf_mode == "text_only":
|
||||
return self._translate_text_only(
|
||||
input_path, output_path, target_language, source_language, progress_callback
|
||||
@@ -294,8 +341,22 @@ class PDFTranslator:
|
||||
original, target_language, source_language
|
||||
)
|
||||
if translated and translated.strip():
|
||||
block["translated"] = translated
|
||||
translated_blocks += 1
|
||||
if translated.strip() == original.strip():
|
||||
# Unchanged (already in the target language, or
|
||||
# the provider failed and returned the source
|
||||
# text). Leave the block completely untouched —
|
||||
# redacting and rewriting identical text would
|
||||
# only degrade its typography (embedded fonts
|
||||
# lost, everything redrawn in the substitute).
|
||||
block["translated"] = None
|
||||
logger.info(
|
||||
"block_translation_unchanged",
|
||||
page=page_num + 1,
|
||||
text_preview=original[:60],
|
||||
)
|
||||
else:
|
||||
block["translated"] = translated
|
||||
translated_blocks += 1
|
||||
else:
|
||||
logger.warning(
|
||||
"block_translation_empty",
|
||||
@@ -357,6 +418,7 @@ class PDFTranslator:
|
||||
return True
|
||||
return False
|
||||
|
||||
redacted_rects: list = []
|
||||
for block in blocks:
|
||||
if block.get("translated"):
|
||||
# Track B3.5: single redaction per block, not per sub-bbox.
|
||||
@@ -371,6 +433,7 @@ class PDFTranslator:
|
||||
page.add_redact_annot(block_bbox, fill=None)
|
||||
else:
|
||||
page.add_redact_annot(block_bbox, fill=(1, 1, 1))
|
||||
redacted_rects.append(block_bbox)
|
||||
|
||||
page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_NONE)
|
||||
|
||||
@@ -378,7 +441,11 @@ class PDFTranslator:
|
||||
# We use the original link geometry (it's unaffected by the
|
||||
# redaction). URIs are preserved verbatim; only the visible
|
||||
# text changes.
|
||||
if page_links_before:
|
||||
# Only links intersecting an actually-redacted block are
|
||||
# re-inserted: when a page has no redaction at all (every block
|
||||
# unchanged), the original annotations survive apply_redactions
|
||||
# and re-inserting would DUPLICATE them.
|
||||
if page_links_before and redacted_rects:
|
||||
reinserted = 0
|
||||
lost = 0
|
||||
page_rect = page.rect
|
||||
@@ -393,6 +460,12 @@ class PDFTranslator:
|
||||
if not page_rect.intersects(from_rect):
|
||||
lost += 1
|
||||
continue
|
||||
# Skip links outside every redacted area — their
|
||||
# original annotation is still alive on the page.
|
||||
if not any(
|
||||
from_rect.intersects(r) for r in redacted_rects
|
||||
):
|
||||
continue
|
||||
# Build the link insertion kwargs based on kind
|
||||
if link.get("uri"):
|
||||
# External URI link
|
||||
@@ -700,6 +773,13 @@ class PDFTranslator:
|
||||
if a.get("_no_merge") or b.get("_no_merge"):
|
||||
return False
|
||||
|
||||
# Table cells: consecutive rows of the same column satisfy every
|
||||
# geometric merge condition (same x0, similar width, small positive
|
||||
# gap) but are UNRELATED values — merging them joins two cells into
|
||||
# one paragraph spanning both rows and breaks the table structure.
|
||||
if a.get("_is_table_cell") or b.get("_is_table_cell"):
|
||||
return False
|
||||
|
||||
# Must have similar font size (within 20%)
|
||||
if abs(a["font_size"] - b["font_size"]) > max(a["font_size"], b["font_size"]) * 0.2:
|
||||
return False
|
||||
@@ -812,8 +892,22 @@ class PDFTranslator:
|
||||
|
||||
# PyMuPDF bug: fontname=None raises AttributeError. Default to 'helv'.
|
||||
# If a custom font file is available, use it via fontfile (fontname ignored).
|
||||
fontname = "helv"
|
||||
fontfile = font_path
|
||||
# Base-14 font variant honouring the block's bold/italic flags —
|
||||
# previously every block (headings included) rendered in regular.
|
||||
# When a custom Unicode fontfile is used we keep it: we have no
|
||||
# bold/italic variant of that file, and glyph coverage matters more
|
||||
# than weight.
|
||||
if font_path:
|
||||
fontname = "helv"
|
||||
fontfile = font_path
|
||||
else:
|
||||
fontname = (
|
||||
"hebi" if (block.get("is_bold") and block.get("is_italic"))
|
||||
else "hebo" if block.get("is_bold")
|
||||
else "heit" if block.get("is_italic")
|
||||
else "helv"
|
||||
)
|
||||
fontfile = None
|
||||
|
||||
# Determine if this is a heading (larger font size = more visual weight)
|
||||
is_heading = target_size >= HEADING_MIN_SIZE
|
||||
@@ -998,6 +1092,12 @@ class PDFTranslator:
|
||||
progress_callback=None,
|
||||
translate_images=translate_images,
|
||||
)
|
||||
# Propagate the inner Word stats so the route's sanity gate
|
||||
# (attempted/changed) works on the fallback path too.
|
||||
for key, value in wt.get_translation_stats().items():
|
||||
self._translation_stats[key] = (
|
||||
self._translation_stats.get(key, 0) + value
|
||||
)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback({
|
||||
@@ -1125,6 +1225,31 @@ class PDFTranslator:
|
||||
pages_text.append(text)
|
||||
doc.close()
|
||||
|
||||
translated_pages = self._translate_page_texts(
|
||||
pages_text, target_language, source_language, progress_callback
|
||||
)
|
||||
|
||||
final_path = output_path.with_suffix(".pdf")
|
||||
self._generate_clean_pdf(translated_pages, final_path, target_language)
|
||||
|
||||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(
|
||||
"pdf_text_only_success",
|
||||
file_name=input_path.name,
|
||||
pages=total_pages,
|
||||
processing_time_ms=processing_time_ms,
|
||||
)
|
||||
|
||||
return final_path
|
||||
|
||||
def _translate_page_texts(
|
||||
self,
|
||||
pages_text: List[str],
|
||||
target_language: str,
|
||||
source_language: str,
|
||||
progress_callback,
|
||||
) -> List[str]:
|
||||
"""Translate per-page texts, keeping order; empty pages pass through."""
|
||||
non_empty_indices = [i for i, t in enumerate(pages_text) if t]
|
||||
|
||||
if progress_callback:
|
||||
@@ -1135,6 +1260,7 @@ class PDFTranslator:
|
||||
})
|
||||
|
||||
translated_pages = list(pages_text)
|
||||
total_pages = len(pages_text)
|
||||
|
||||
for seq, page_idx in enumerate(non_empty_indices):
|
||||
text = pages_text[page_idx]
|
||||
@@ -1160,19 +1286,139 @@ class PDFTranslator:
|
||||
"progress_override": pct,
|
||||
})
|
||||
|
||||
return translated_pages
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# SCANNED PDFs — Mistral OCR
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _is_scanned_pdf(self, input_path: Path) -> bool:
|
||||
"""True when the PDF is image-only (no usable text layer).
|
||||
|
||||
A page counts as a scan page when it has almost no extractable
|
||||
text AND is mostly covered by raster images. The document is
|
||||
considered scanned when it contains scan pages and no page with a
|
||||
real text layer — this keeps sparse-but-textual PDFs (a bare
|
||||
title page, a single label) on the normal layout pipeline.
|
||||
"""
|
||||
from config import config
|
||||
|
||||
try:
|
||||
import fitz
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
try:
|
||||
with fitz.open(str(input_path)) as doc:
|
||||
if len(doc) == 0:
|
||||
return False
|
||||
has_text_page = False
|
||||
has_scan_page = False
|
||||
for page in doc:
|
||||
if len(page.get_text("text").strip()) >= config.SCANNED_PDF_MIN_CHARS_PER_PAGE:
|
||||
has_text_page = True
|
||||
continue
|
||||
# Text-poor page: a scan page only when raster images
|
||||
# cover most of its area (a bare title page has none).
|
||||
page_area = abs(page.rect)
|
||||
img_area = 0.0
|
||||
for img in page.get_images(full=True):
|
||||
for rect in page.get_image_rects(img[0]):
|
||||
img_area += abs(rect & page.rect)
|
||||
if page_area and img_area / page_area >= 0.5:
|
||||
has_scan_page = True
|
||||
except Exception as e:
|
||||
logger.warning("scanned_pdf_detection_failed", error=str(e))
|
||||
return False
|
||||
|
||||
scanned = has_scan_page and not has_text_page
|
||||
if scanned:
|
||||
logger.info(
|
||||
"scanned_pdf_detected",
|
||||
file=input_path.name,
|
||||
)
|
||||
return scanned
|
||||
|
||||
def _translate_scanned_pdf(
|
||||
self,
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
target_language: str,
|
||||
source_language: str,
|
||||
progress_callback,
|
||||
) -> Path:
|
||||
"""OCR (Mistral) → translate → clean re-typeset PDF.
|
||||
|
||||
The source pages are images, so the original layout cannot be
|
||||
rewritten in place; the output carries the recovered text in a
|
||||
clean document instead.
|
||||
"""
|
||||
from config import config
|
||||
from services.mistral_ocr import MistralOCRClient, MistralOCRError
|
||||
|
||||
# Resolution order: admin settings (via set_ocr_config) > env/config.
|
||||
api_key = (self._ocr_api_key or "").strip() or config.MISTRAL_API_KEY
|
||||
model = (self._ocr_model or "").strip() or config.MISTRAL_OCR_MODEL
|
||||
timeout = self._ocr_timeout or config.MISTRAL_OCR_TIMEOUT
|
||||
ocr_enabled = (
|
||||
config.MISTRAL_OCR_ENABLED
|
||||
if self._ocr_enabled is None
|
||||
else self._ocr_enabled
|
||||
)
|
||||
|
||||
if not ocr_enabled or not api_key:
|
||||
raise RuntimeError(
|
||||
"PDF scanné détecté (pages image sans couche texte). "
|
||||
"La traduction des PDF scannés nécessite l'OCR Mistral : "
|
||||
"configurez MISTRAL_API_KEY (ou fournissez un PDF avec du texte sélectionnable)."
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
client = MistralOCRClient(
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
pages_markdown = client.extract_pdf_text(
|
||||
input_path, progress_callback=progress_callback
|
||||
)
|
||||
pages_text = [self._markdown_to_text(md) for md in pages_markdown]
|
||||
|
||||
translated_pages = self._translate_page_texts(
|
||||
pages_text, target_language, source_language, progress_callback
|
||||
)
|
||||
|
||||
final_path = output_path.with_suffix(".pdf")
|
||||
self._generate_clean_pdf(translated_pages, final_path, target_language)
|
||||
|
||||
processing_time_ms = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(
|
||||
"pdf_text_only_success",
|
||||
"pdf_scanned_success",
|
||||
file_name=input_path.name,
|
||||
pages=total_pages,
|
||||
pages=len(pages_text),
|
||||
processing_time_ms=processing_time_ms,
|
||||
)
|
||||
|
||||
return final_path
|
||||
|
||||
@staticmethod
|
||||
def _markdown_to_text(markdown: str) -> str:
|
||||
"""Flatten OCR markdown to plain text (drop images/links/markup)."""
|
||||
import re
|
||||
|
||||
if not markdown:
|
||||
return ""
|
||||
text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", markdown) # images
|
||||
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) # links → label
|
||||
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) # headings
|
||||
text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) # bullets
|
||||
# Markdown table rows → plain line of cells
|
||||
text = re.sub(r"^\s*\|", "", text, flags=re.MULTILINE)
|
||||
text = text.replace("|", " ")
|
||||
text = re.sub(r"^\s*[-:| ]+\s*$", "", text, flags=re.MULTILINE) # rules
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
def _generate_clean_pdf(
|
||||
self, pages_text: List[str], output_path: Path, target_language: str = "en"
|
||||
) -> None:
|
||||
@@ -1305,18 +1551,29 @@ class PDFTranslator:
|
||||
def _translate_single(
|
||||
self, text: str, target_language: str, source_language: str
|
||||
) -> str:
|
||||
"""Translate a single text string."""
|
||||
"""Translate a single text string.
|
||||
|
||||
Also feeds the job-level attempted/changed stats so the route can
|
||||
detect a total provider failure (changed == 0) on PDFs too.
|
||||
"""
|
||||
if text and text.strip():
|
||||
self._translation_stats["attempted"] += 1
|
||||
if self._provider is not None:
|
||||
try:
|
||||
results = self._translate_with_provider([text], target_language, source_language)
|
||||
if results and results[0].strip():
|
||||
if results[0].strip() != text.strip():
|
||||
self._translation_stats["changed"] += 1
|
||||
return results[0]
|
||||
except Exception as e:
|
||||
logger.warning("provider_single_failed", error=str(e))
|
||||
|
||||
from services.translation_service import translation_service
|
||||
try:
|
||||
return translation_service.translate_text(text, target_language, source_language)
|
||||
result = translation_service.translate_text(text, target_language, source_language)
|
||||
if result and result.strip() and result.strip() != text.strip():
|
||||
self._translation_stats["changed"] += 1
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning("legacy_single_failed", error=str(e))
|
||||
return text
|
||||
|
||||
@@ -95,6 +95,69 @@ def _apply_rtl_to_shape(shape) -> None:
|
||||
_apply_rtl_to_shape(sub_shape)
|
||||
|
||||
|
||||
# East-Asian typeface hints per target language (DrawingML <a:ea>)
|
||||
_EA_TYPEFACES = {
|
||||
"zh": "SimSun",
|
||||
"zh-CN": "SimSun",
|
||||
"zh-TW": "PMingLiU",
|
||||
"ja": "Yu Mincho",
|
||||
"ko": "Batang",
|
||||
}
|
||||
|
||||
|
||||
def _ea_typeface_for_target(target_language: str):
|
||||
code = (target_language or "").strip()
|
||||
base = code.split("-")[0].lower()
|
||||
return _EA_TYPEFACES.get(code) or _EA_TYPEFACES.get(base)
|
||||
|
||||
|
||||
def _apply_ea_font_hints(presentation: Presentation, target_language: str) -> None:
|
||||
"""Set the <a:ea> (east-asian) typeface on every run for CJK targets.
|
||||
|
||||
Blanket application is safe — the hint only affects CJK glyphs.
|
||||
"""
|
||||
|
||||
def _hint_shape(shape) -> int:
|
||||
hinted = 0
|
||||
if shape.has_text_frame:
|
||||
hinted += _hint_text_frame(shape.text_frame)
|
||||
if shape.shape_type == MSO_SHAPE_TYPE.TABLE:
|
||||
for row in shape.table.rows:
|
||||
for cell in row.cells:
|
||||
hinted += _hint_text_frame(cell.text_frame)
|
||||
if shape.shape_type == MSO_SHAPE_TYPE.GROUP:
|
||||
for sub_shape in shape.shapes:
|
||||
hinted += _hint_shape(sub_shape)
|
||||
return hinted
|
||||
|
||||
def _hint_text_frame(text_frame) -> int:
|
||||
hinted = 0
|
||||
tag_rPr = f"{{{_NS_A}}}rPr"
|
||||
tag_ea = f"{{{_NS_A}}}ea"
|
||||
for paragraph in text_frame.paragraphs:
|
||||
for run in paragraph.runs:
|
||||
rPr = run._r.find(tag_rPr)
|
||||
if rPr is None:
|
||||
rPr = etree.SubElement(run._r, tag_rPr)
|
||||
ea = rPr.find(tag_ea)
|
||||
if ea is None:
|
||||
ea = etree.SubElement(rPr, tag_ea)
|
||||
if not ea.get("typeface"):
|
||||
ea.set("typeface", typeface)
|
||||
hinted += 1
|
||||
return hinted
|
||||
|
||||
typeface = _ea_typeface_for_target(target_language)
|
||||
if not typeface:
|
||||
return
|
||||
total = 0
|
||||
for slide in presentation.slides:
|
||||
for shape in slide.shapes:
|
||||
total += _hint_shape(shape)
|
||||
if total:
|
||||
_log_info("pptx_ea_font_hints_applied", runs=total, typeface=typeface)
|
||||
|
||||
|
||||
class PptxProcessorError(Exception):
|
||||
"""Exception for PowerPoint processing errors with structured error codes."""
|
||||
|
||||
@@ -152,6 +215,7 @@ class PowerPointTranslator:
|
||||
"""
|
||||
self._provider = provider
|
||||
self._custom_prompt: Optional[str] = None
|
||||
self._tm_scope = None # set via set_tm_scope (per-user translation memory)
|
||||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||||
|
||||
def set_provider(self, provider: TranslationProvider) -> None:
|
||||
@@ -161,6 +225,14 @@ class PowerPointTranslator:
|
||||
def set_custom_prompt(self, prompt: Optional[str]) -> None:
|
||||
"""Set custom system prompt for LLM providers."""
|
||||
self._custom_prompt = prompt
|
||||
def set_tm_scope(self, user_id, prompt=None) -> None:
|
||||
"""Enable the per-user translation memory for this job."""
|
||||
from services.translation_tm import TMScope
|
||||
|
||||
self._tm_scope = TMScope.from_prompt(
|
||||
user_id, prompt or getattr(self, "_custom_prompt", None)
|
||||
)
|
||||
|
||||
|
||||
def translate_file(
|
||||
self,
|
||||
@@ -316,6 +388,10 @@ class PowerPointTranslator:
|
||||
if target_language.lower() in RTL_LANGUAGES:
|
||||
_apply_rtl_to_presentation(presentation)
|
||||
|
||||
# CJK font hint so the target script renders with a proper
|
||||
# typeface instead of shape-dependent fallbacks.
|
||||
_apply_ea_font_hints(presentation, target_language)
|
||||
|
||||
if translate_images:
|
||||
try:
|
||||
self._translate_images(presentation, target_language)
|
||||
@@ -405,12 +481,30 @@ class PowerPointTranslator:
|
||||
non_empty = [t for t in texts if t and t.strip()]
|
||||
self._translation_stats["attempted"] += len(non_empty)
|
||||
|
||||
from services.translation_tm import translate_with_tm
|
||||
|
||||
provider_name = (
|
||||
self._provider.get_name() if hasattr(self._provider, "get_name")
|
||||
else type(self._provider).__name__
|
||||
) if self._provider is not None else "legacy"
|
||||
|
||||
if self._provider is not None:
|
||||
translated = self._translate_with_provider(
|
||||
texts, target_language, source_language
|
||||
)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_provider(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
else:
|
||||
translated = self._translate_with_legacy(texts, target_language, source_language)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_legacy(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
|
||||
# Translation memory: reuse this user's previous translations
|
||||
# (identical context/prompt) before hitting the provider.
|
||||
translated = translate_with_tm(
|
||||
texts, target_language, source_language,
|
||||
provider_name, getattr(self, "_tm_scope", None), _do_translate,
|
||||
)
|
||||
|
||||
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
|
||||
self._translation_stats["changed"] += changed
|
||||
@@ -743,7 +837,14 @@ class PowerPointTranslator:
|
||||
return None
|
||||
|
||||
def _apply_chart_translations(self, output_path: Path) -> None:
|
||||
"""Re-inject chart text translations by modifying chart XML parts.
|
||||
"""Re-inject chart text translations into the saved .pptx ZIP.
|
||||
|
||||
python-pptx's ChartPart exposes a read-only ``blob`` property, so an
|
||||
in-memory `chart_part.blob = ...` assignment fails silently and the
|
||||
chart text never reaches the output file. Instead — exactly like the
|
||||
Word translator — we translate into a fresh parse of each chart
|
||||
part's XML and rewrite the corresponding ZIP entries of the
|
||||
already-saved output file.
|
||||
|
||||
Matching strategy: prefer the stored `element_path` (set at collect
|
||||
time) to navigate directly to the right element. Fall back to
|
||||
@@ -759,6 +860,9 @@ class PowerPointTranslator:
|
||||
total_translated = 0
|
||||
total_skipped = 0
|
||||
|
||||
# partname (e.g. "ppt/charts/chart1.xml") → updated XML bytes
|
||||
updated_parts: Dict[str, bytes] = {}
|
||||
|
||||
for chart_data in self._chart_entries:
|
||||
entries = chart_data['entries']
|
||||
chart_part = chart_data['chart_part']
|
||||
@@ -804,17 +908,39 @@ class PowerPointTranslator:
|
||||
target.text = leading + (entry['translated'] or '').strip() + trailing
|
||||
total_translated += 1
|
||||
|
||||
# Update the chart part blob
|
||||
chart_part.blob = etree.tostring(
|
||||
chart_xml,
|
||||
xml_declaration=True,
|
||||
encoding='UTF-8',
|
||||
standalone=True,
|
||||
)
|
||||
part_name = str(getattr(chart_part, "partname", "")).lstrip("/")
|
||||
if part_name:
|
||||
updated_parts[part_name] = etree.tostring(
|
||||
chart_xml,
|
||||
xml_declaration=True,
|
||||
encoding='UTF-8',
|
||||
standalone=True,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
_log_error("pptx_chart_update_error", error=str(e))
|
||||
|
||||
# Single ZIP rewrite pass for all updated chart parts
|
||||
if updated_parts:
|
||||
import zipfile
|
||||
import io as _io
|
||||
try:
|
||||
with zipfile.ZipFile(output_path, 'r') as zf_in:
|
||||
buf = _io.BytesIO()
|
||||
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf_out:
|
||||
for item in zf_in.namelist():
|
||||
data = updated_parts.get(item, zf_in.read(item))
|
||||
zf_out.writestr(item, data)
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(buf.getvalue())
|
||||
_log_info(
|
||||
"pptx_charts_rewritten",
|
||||
chart_parts=len(updated_parts),
|
||||
translated=total_translated,
|
||||
)
|
||||
except Exception as e:
|
||||
_log_error("pptx_chart_zip_rewrite_error", error=str(e))
|
||||
|
||||
# Clean up
|
||||
self._chart_entries = []
|
||||
|
||||
|
||||
@@ -31,6 +31,73 @@ RTL_LANGUAGES: frozenset = frozenset(
|
||||
{"ar", "he", "fa", "ur", "ku", "ps", "ug", "sd", "yi", "dv", "ckb"}
|
||||
)
|
||||
|
||||
# East-Asian / complex-script font hints: when the target language uses
|
||||
# glyphs a Latin theme font lacks, Word falls back to a substitute —
|
||||
# setting the eastAsia (CJK) or cs (Arabic script) typeface keeps the
|
||||
# rendering consistent across runs.
|
||||
CJK_EASTASIA_FONTS: dict = {
|
||||
"zh": "SimSun",
|
||||
"zh-CN": "SimSun",
|
||||
"zh-TW": "PMingLiU",
|
||||
"ja": "Yu Mincho",
|
||||
"ko": "Batang",
|
||||
}
|
||||
CS_FONTS: dict = {
|
||||
"ar": "Arial",
|
||||
"he": "Arial",
|
||||
"fa": "Arial",
|
||||
"ur": "Arial",
|
||||
}
|
||||
|
||||
|
||||
def _font_hints_for_target(target_language: str):
|
||||
"""(eastAsia_font, cs_font) hints for the target language, if any."""
|
||||
code = (target_language or "").strip()
|
||||
base = code.split("-")[0].lower()
|
||||
return CJK_EASTASIA_FONTS.get(code) or CJK_EASTASIA_FONTS.get(base), CS_FONTS.get(base)
|
||||
|
||||
|
||||
def _apply_font_hints(document: Document, target_language: str) -> None:
|
||||
"""Set eastAsia/cs typeface hints on every run for CJK/Arabic targets.
|
||||
|
||||
Blanket application is safe: the hint only affects the glyphs of that
|
||||
script, which Latin text does not contain.
|
||||
"""
|
||||
eastasia, cs = _font_hints_for_target(target_language)
|
||||
if not eastasia and not cs:
|
||||
return
|
||||
|
||||
runs = []
|
||||
for para in document.paragraphs:
|
||||
runs.extend(para.runs)
|
||||
for table in document.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
for para in cell.paragraphs:
|
||||
runs.extend(para.runs)
|
||||
for section in document.sections:
|
||||
for hf in (section.header, section.footer):
|
||||
for para in hf.paragraphs:
|
||||
runs.extend(para.runs)
|
||||
|
||||
hinted = 0
|
||||
for run in runs:
|
||||
rPr = run._r.get_or_add_rPr()
|
||||
rFonts = rPr.find(qn("w:rFonts"))
|
||||
if rFonts is None:
|
||||
rFonts = OxmlElement("w:rFonts")
|
||||
rPr.insert(0, rFonts)
|
||||
if eastasia and not rFonts.get(qn("w:eastAsia")):
|
||||
rFonts.set(qn("w:eastAsia"), eastasia)
|
||||
hinted += 1
|
||||
if cs and not rFonts.get(qn("w:cs")):
|
||||
rFonts.set(qn("w:cs"), cs)
|
||||
hinted += 1
|
||||
|
||||
if hinted:
|
||||
from core.logging import get_logger as _gl
|
||||
_gl(__name__).info("word_font_hints_applied", runs=hinted, eastasia=eastasia, cs=cs)
|
||||
|
||||
|
||||
from core.logging import get_logger
|
||||
|
||||
@@ -62,7 +129,9 @@ def _set_paragraph_rtl(paragraph: Paragraph) -> None:
|
||||
|
||||
Sets:
|
||||
- w:pPr/w:bidi → paragraph text direction = RTL
|
||||
- w:pPr/w:jc → alignment = right
|
||||
- w:pPr/w:jc → mirrored alignment (left→right), ONLY when the
|
||||
paragraph has no explicit alignment — centered/justified titles
|
||||
must not be forced right-aligned.
|
||||
- w:rPr/w:rtl → run-level RTL marker for each run
|
||||
"""
|
||||
pPr = paragraph._p.get_or_add_pPr()
|
||||
@@ -71,10 +140,12 @@ def _set_paragraph_rtl(paragraph: Paragraph) -> None:
|
||||
pPr.append(OxmlElement("w:bidi"))
|
||||
|
||||
jc = pPr.find(qn("w:jc"))
|
||||
if jc is None:
|
||||
jc = OxmlElement("w:jc")
|
||||
pPr.append(jc)
|
||||
jc.set(qn("w:val"), "right")
|
||||
explicit_alignment = jc is not None and jc.get(qn("w:val")) not in (None, "", "left")
|
||||
if not explicit_alignment:
|
||||
if jc is None:
|
||||
jc = OxmlElement("w:jc")
|
||||
pPr.append(jc)
|
||||
jc.set(qn("w:val"), "right")
|
||||
|
||||
for run in paragraph.runs:
|
||||
rPr = run._r.get_or_add_rPr()
|
||||
@@ -173,6 +244,7 @@ class WordTranslator:
|
||||
self._provider = provider
|
||||
self._custom_prompt: Optional[str] = None
|
||||
self._translation_stats = {"attempted": 0, "changed": 0}
|
||||
self._tm_scope = None # set via set_tm_scope (per-user translation memory)
|
||||
|
||||
def set_provider(self, provider: TranslationProvider) -> None:
|
||||
"""Set the translation provider."""
|
||||
@@ -182,6 +254,12 @@ class WordTranslator:
|
||||
"""Set custom system prompt for LLM providers."""
|
||||
self._custom_prompt = prompt
|
||||
|
||||
def set_tm_scope(self, user_id: Optional[str], prompt: Optional[str] = None) -> None:
|
||||
"""Enable the per-user translation memory for this job."""
|
||||
from services.translation_tm import TMScope
|
||||
|
||||
self._tm_scope = TMScope.from_prompt(user_id, prompt or self._custom_prompt)
|
||||
|
||||
def translate_file(
|
||||
self,
|
||||
input_path: Path,
|
||||
@@ -333,6 +411,10 @@ class WordTranslator:
|
||||
if target_language.lower() in RTL_LANGUAGES:
|
||||
_apply_rtl_to_document(document)
|
||||
|
||||
# CJK / Arabic-script font hints so Word renders the target
|
||||
# script with a proper typeface instead of per-run fallbacks.
|
||||
_apply_font_hints(document, target_language)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
{
|
||||
@@ -461,12 +543,30 @@ class WordTranslator:
|
||||
non_empty = [t for t in texts if t and t.strip()]
|
||||
self._translation_stats["attempted"] += len(non_empty)
|
||||
|
||||
from services.translation_tm import translate_with_tm
|
||||
|
||||
provider_name = (
|
||||
self._provider.get_name() if hasattr(self._provider, "get_name")
|
||||
else type(self._provider).__name__
|
||||
) if self._provider is not None else "legacy"
|
||||
|
||||
if self._provider is not None:
|
||||
translated = self._translate_with_provider(
|
||||
texts, target_language, source_language
|
||||
)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_provider(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
else:
|
||||
translated = self._translate_with_legacy(texts, target_language, source_language)
|
||||
def _do_translate(miss_texts):
|
||||
return self._translate_with_legacy(
|
||||
miss_texts, target_language, source_language
|
||||
)
|
||||
|
||||
# Translation memory: reuse this user's previous translations
|
||||
# (identical context/prompt) before hitting the provider.
|
||||
translated = translate_with_tm(
|
||||
texts, target_language, source_language,
|
||||
provider_name, self._tm_scope, _do_translate,
|
||||
)
|
||||
|
||||
changed = sum(1 for orig, trans in zip(texts, translated) if orig != trans and trans.strip())
|
||||
self._translation_stats["changed"] += changed
|
||||
@@ -541,12 +641,19 @@ class WordTranslator:
|
||||
|
||||
Handles: paragraphs, tables, SDT (TOC/index), text boxes, shapes,
|
||||
AlternateContent blocks, and any nested drawing elements.
|
||||
|
||||
A single ``seen_run_elements`` set is shared by every collector so
|
||||
runs living in text boxes are never collected twice (the paragraph
|
||||
walk descends into w:txbxContent too).
|
||||
"""
|
||||
count_before = len(text_elements)
|
||||
seen_run_elements: set = set()
|
||||
|
||||
# Pass 1: walk direct body children
|
||||
for element in document.element.body:
|
||||
self._collect_from_element(element, document, text_elements)
|
||||
self._collect_from_element(
|
||||
element, document, text_elements, seen_run_elements
|
||||
)
|
||||
|
||||
pass1_count = len(text_elements) - count_before
|
||||
|
||||
@@ -554,15 +661,18 @@ class WordTranslator:
|
||||
# Text boxes / rectangles / shapes store their text here, nested deep
|
||||
# inside <w:drawing> → <a:graphic> → <wps:wsp> → <wps:txbx> or
|
||||
# inside <w:pict> → <v:shape> → <v:textbox>.
|
||||
self._collect_from_textboxes(document.element.body, document, text_elements)
|
||||
self._collect_from_textboxes(
|
||||
document.element.body, document, text_elements, seen_run_elements
|
||||
)
|
||||
|
||||
pass2_count = len(text_elements) - count_before - pass1_count
|
||||
|
||||
# Pass 3: footnotes and endnotes (live in separate parts)
|
||||
# Pass 3: footnotes, endnotes and comments (live in separate parts)
|
||||
if post_save_callbacks is None:
|
||||
post_save_callbacks = []
|
||||
self._collect_from_footnotes(document, text_elements, post_save_callbacks)
|
||||
self._collect_from_endnotes(document, text_elements, post_save_callbacks)
|
||||
self._collect_from_comments(document, text_elements, post_save_callbacks)
|
||||
|
||||
total = len(text_elements) - count_before
|
||||
_log_info(
|
||||
@@ -573,34 +683,38 @@ class WordTranslator:
|
||||
)
|
||||
|
||||
def _collect_from_element(
|
||||
self, element, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
self, element, document: Document,
|
||||
text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
seen_run_elements: Optional[set] = None,
|
||||
) -> None:
|
||||
"""Recursively collect from any element type."""
|
||||
if isinstance(element, CT_P):
|
||||
paragraph = Paragraph(element, document)
|
||||
self._collect_from_paragraph(paragraph, text_elements)
|
||||
self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
|
||||
elif isinstance(element, CT_Tbl):
|
||||
table = Table(element, document)
|
||||
self._collect_from_table(table, text_elements)
|
||||
self._collect_from_table(table, text_elements, seen_run_elements)
|
||||
elif element.tag == qn("w:sdt"):
|
||||
self._collect_from_sdt(element, document, text_elements)
|
||||
self._collect_from_sdt(element, document, text_elements, seen_run_elements)
|
||||
elif element.tag == self._TAG_ALT_CONTENT:
|
||||
# <mc:AlternateContent> wraps drawing/shape content
|
||||
for part in element:
|
||||
self._collect_from_element(part, document, text_elements)
|
||||
self._collect_from_element(part, document, text_elements, seen_run_elements)
|
||||
else:
|
||||
# For any other container element, recurse into children
|
||||
# to catch paragraphs nested in unexpected wrappers
|
||||
for child in element:
|
||||
if isinstance(child, CT_P):
|
||||
paragraph = Paragraph(child, document)
|
||||
self._collect_from_paragraph(paragraph, text_elements)
|
||||
self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
|
||||
elif isinstance(child, CT_Tbl):
|
||||
table = Table(child, document)
|
||||
self._collect_from_table(table, text_elements)
|
||||
self._collect_from_table(table, text_elements, seen_run_elements)
|
||||
|
||||
def _collect_from_textboxes(
|
||||
self, root, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
self, root, document: Document,
|
||||
text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
seen_run_elements: Optional[set] = None,
|
||||
) -> None:
|
||||
"""Find and collect text from ALL <w:txbxContent> elements in the XML tree.
|
||||
|
||||
@@ -612,20 +726,23 @@ class WordTranslator:
|
||||
- Shapes nested in <mc:AlternateContent> blocks
|
||||
|
||||
The <w:txbxContent> element contains regular <w:p> paragraphs
|
||||
with <w:r> runs, just like normal body text.
|
||||
with <w:r> runs, just like normal body text. Runs already collected
|
||||
during the body walk are skipped via ``seen_run_elements``.
|
||||
"""
|
||||
# Find all w:txbxContent elements anywhere in the tree
|
||||
for txbx in root.iter(qn("w:txbxContent")):
|
||||
for child in txbx:
|
||||
if isinstance(child, CT_P):
|
||||
paragraph = Paragraph(child, document)
|
||||
self._collect_from_paragraph(paragraph, text_elements)
|
||||
self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
|
||||
elif isinstance(child, CT_Tbl):
|
||||
table = Table(child, document)
|
||||
self._collect_from_table(table, text_elements)
|
||||
self._collect_from_table(table, text_elements, seen_run_elements)
|
||||
|
||||
def _collect_from_sdt(
|
||||
self, sdt_element, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
self, sdt_element, document: Document,
|
||||
text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
seen_run_elements: Optional[set] = None,
|
||||
) -> None:
|
||||
"""Collect text from Structured Document Tags (TOC, index, content controls).
|
||||
|
||||
@@ -645,10 +762,10 @@ class WordTranslator:
|
||||
for child in sdt_content:
|
||||
if isinstance(child, CT_P):
|
||||
paragraph = Paragraph(child, document)
|
||||
self._collect_from_paragraph(paragraph, text_elements)
|
||||
self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
|
||||
elif isinstance(child, CT_Tbl):
|
||||
table = Table(child, document)
|
||||
self._collect_from_table(table, text_elements)
|
||||
self._collect_from_table(table, text_elements, seen_run_elements)
|
||||
|
||||
def _collect_from_footnotes(
|
||||
self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
@@ -783,6 +900,59 @@ class WordTranslator:
|
||||
|
||||
post_save_callbacks.append(write_endnotes_back)
|
||||
|
||||
def _collect_from_comments(
|
||||
self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
post_save_callbacks: List[Callable[[Path], None]] = None,
|
||||
) -> None:
|
||||
"""Collect text from comments/balloons (word/comments.xml part).
|
||||
|
||||
Same mechanism as footnotes: the comments part is separate from the
|
||||
main document tree, so translations are written back after save.
|
||||
"""
|
||||
comments_xml = self._find_part_by_content_type(
|
||||
document,
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml",
|
||||
)
|
||||
if comments_xml is None:
|
||||
return
|
||||
|
||||
collected = 0
|
||||
for t_elem in comments_xml.iter(qn("w:t")):
|
||||
original = t_elem.text or ""
|
||||
if not original.strip():
|
||||
continue
|
||||
|
||||
def make_t_setter(t):
|
||||
def setter(text: str) -> None:
|
||||
t.text = text
|
||||
return setter
|
||||
|
||||
text_elements.append((original, make_t_setter(t_elem)))
|
||||
collected += 1
|
||||
|
||||
if collected and post_save_callbacks is not None:
|
||||
def write_comments_back(output_path: Path) -> None:
|
||||
try:
|
||||
new_blob = etree.tostring(
|
||||
comments_xml,
|
||||
xml_declaration=True,
|
||||
encoding="UTF-8",
|
||||
standalone=True,
|
||||
)
|
||||
tmp_path = output_path.with_suffix(".tmp_com")
|
||||
with zipfile.ZipFile(output_path, "r") as zin, \
|
||||
zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zout:
|
||||
for item in zin.namelist():
|
||||
if item == "word/comments.xml":
|
||||
zout.writestr(item, new_blob)
|
||||
else:
|
||||
zout.writestr(item, zin.read(item))
|
||||
tmp_path.replace(output_path)
|
||||
except Exception as e:
|
||||
_log_error("word_comments_writeback_error", error=str(e))
|
||||
|
||||
post_save_callbacks.append(write_comments_back)
|
||||
|
||||
def _collect_from_charts(
|
||||
self, document: Document, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
) -> None:
|
||||
@@ -1144,23 +1314,39 @@ class WordTranslator:
|
||||
except Exception as e:
|
||||
_log_error("word_diagram_zip_rewrite_error", error=str(e))
|
||||
|
||||
@staticmethod
|
||||
def _rpr_signature(run_element) -> str:
|
||||
"""Formatting signature of a run: serialized rPr XML (or "")."""
|
||||
rpr = run_element.find(qn("w:rPr"))
|
||||
if rpr is None:
|
||||
return ""
|
||||
import lxml.etree as _et
|
||||
|
||||
return _et.tostring(rpr, encoding="unicode")
|
||||
|
||||
def _collect_from_paragraph(
|
||||
self,
|
||||
paragraph: Paragraph,
|
||||
text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
seen_run_elements: Optional[set] = None,
|
||||
) -> None:
|
||||
"""Collect text from paragraph runs, preserving inter-run whitespace.
|
||||
|
||||
Each run is sent for translation WITHOUT its surrounding whitespace.
|
||||
The whitespace is captured and reapplied after translation so that words
|
||||
at formatting boundaries (e.g. bold/normal) do not get concatenated.
|
||||
Adjacent runs sharing the SAME parent element and the SAME run
|
||||
formatting (rPr) are merged into ONE translation unit: the sentence
|
||||
is translated whole — not fragment by fragment — and the result is
|
||||
written into the first run while the sibling runs are blanked.
|
||||
This is what keeps mid-sentence bold spans ("This is *very*
|
||||
important") coherent in the target language, like DeepL's inline
|
||||
tag handling.
|
||||
|
||||
Note: python-docx's `paragraph.runs` only returns DIRECT child <w:r>
|
||||
elements, not those inside <w:hyperlink> (used for TOC entries,
|
||||
cross-references, bookmark links). We therefore iterate the full
|
||||
XML tree to find every <w:r> and use a set of element ids to
|
||||
deduplicate — this avoids translating the same run twice while
|
||||
ensuring hyperlink text IS picked up.
|
||||
XML tree to find every <w:r> and deduplicate by element identity —
|
||||
`seen_run_elements` is shared across paragraphs so runs living in
|
||||
text boxes (also collected by _collect_from_textboxes) are not
|
||||
translated twice.
|
||||
"""
|
||||
# Check full paragraph text including nested content (hyperlinks, etc.)
|
||||
full_text = ''.join(
|
||||
@@ -1169,33 +1355,83 @@ class WordTranslator:
|
||||
if not full_text:
|
||||
return
|
||||
|
||||
# Collect every <w:r> element in the paragraph tree, including
|
||||
# those nested in <w:hyperlink>, <w:smartTag>, etc. The dedup by
|
||||
# element id is defensive — `paragraph.runs` and the manual iter
|
||||
# below could overlap if python-docx starts surfacing nested runs.
|
||||
seen_run_ids: set = set()
|
||||
if seen_run_elements is None:
|
||||
seen_run_elements = set()
|
||||
|
||||
# 1) Direct runs (paragraph.runs is the python-docx-native API).
|
||||
for run in paragraph.runs:
|
||||
run_id = id(run._r)
|
||||
if run_id in seen_run_ids:
|
||||
continue
|
||||
seen_run_ids.add(run_id)
|
||||
if run.text and run.text.strip():
|
||||
self._append_run_translation(run, text_elements)
|
||||
|
||||
# 2) Runs nested inside <w:hyperlink> (TOC, cross-references).
|
||||
# python-docx's `paragraph.runs` does NOT descend into hyperlinks in
|
||||
# version 1.x — we have to walk the XML ourselves.
|
||||
# Every <w:r> in the paragraph tree, in document order, deduplicated
|
||||
# by element identity (paragraph.runs and the manual iter overlap).
|
||||
ordered_runs = []
|
||||
for r_elem in paragraph._p.iter(qn('w:r')):
|
||||
run_id = id(r_elem)
|
||||
if run_id in seen_run_ids:
|
||||
if id(r_elem) in seen_run_elements:
|
||||
continue
|
||||
seen_run_ids.add(run_id)
|
||||
# Build a Run wrapper so the setter API is consistent.
|
||||
run = Run(r_elem, paragraph)
|
||||
if run.text and run.text.strip():
|
||||
seen_run_elements.add(id(r_elem))
|
||||
ordered_runs.append(r_elem)
|
||||
|
||||
# Merge adjacent runs: same parent + same formatting signature.
|
||||
# Merging never crosses a parent boundary, so runs belonging to
|
||||
# different hyperlinks stay separate units.
|
||||
group: list = [] # list of r_elems
|
||||
group_signature: Optional[str] = None
|
||||
|
||||
def _flush_group():
|
||||
combined = "".join(
|
||||
(t.text or "")
|
||||
for r in group
|
||||
for t in r.findall(qn("w:t"))
|
||||
)
|
||||
if not combined.strip():
|
||||
return
|
||||
non_empty = [r for r in group if r.findall(qn("w:t"))]
|
||||
if len(non_empty) == 1:
|
||||
run = Run(non_empty[0], paragraph)
|
||||
self._append_run_translation(run, text_elements)
|
||||
return
|
||||
leading = combined[: len(combined) - len(combined.lstrip())]
|
||||
trailing = combined[len(combined.rstrip()):]
|
||||
stripped = combined.strip()
|
||||
if not stripped:
|
||||
return
|
||||
|
||||
first = non_empty[0]
|
||||
|
||||
def make_group_setter(first_r, siblings, lead: str, trail: str):
|
||||
def setter(text: str) -> None:
|
||||
from docx.text.run import Run as _Run
|
||||
|
||||
run = _Run(first_r, paragraph)
|
||||
# Reapply the group's boundary whitespace so words are
|
||||
# never concatenated with the next differently-formatted
|
||||
# run ("This is quite" + "very" → "quite very").
|
||||
run.text = lead + text.strip() + trail
|
||||
# Blank the merged siblings: the whole sentence now
|
||||
# lives in the first run (formatting is identical).
|
||||
for sib in siblings:
|
||||
for t_elem in sib.findall(qn("w:t")):
|
||||
t_elem.text = ""
|
||||
|
||||
return setter
|
||||
|
||||
siblings = non_empty[1:]
|
||||
text_elements.append(
|
||||
(stripped, make_group_setter(first, siblings, leading, trailing))
|
||||
)
|
||||
|
||||
for r_elem in ordered_runs:
|
||||
# Whitespace-only runs join the group: dropping them would
|
||||
# concatenate words ("Hello" + " " + "World" → "HelloWorld").
|
||||
# They carry no w:t text, so a group of only whitespace runs is
|
||||
# skipped at flush time by the strip() check.
|
||||
signature = self._rpr_signature(r_elem)
|
||||
same_parent = (
|
||||
group and group[-1].getparent() is r_elem.getparent()
|
||||
)
|
||||
if group and same_parent and signature == group_signature:
|
||||
group.append(r_elem)
|
||||
else:
|
||||
_flush_group()
|
||||
group = [r_elem]
|
||||
group_signature = signature
|
||||
_flush_group()
|
||||
|
||||
def _append_run_translation(
|
||||
self,
|
||||
@@ -1220,15 +1456,16 @@ class WordTranslator:
|
||||
text_elements.append((stripped, make_setter(run, leading, trailing)))
|
||||
|
||||
def _collect_from_table(
|
||||
self, table: Table, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
self, table: Table, text_elements: List[Tuple[str, Callable[[str], None]]],
|
||||
seen_run_elements: Optional[set] = None,
|
||||
) -> None:
|
||||
"""Collect text from table cells."""
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
for paragraph in cell.paragraphs:
|
||||
self._collect_from_paragraph(paragraph, text_elements)
|
||||
self._collect_from_paragraph(paragraph, text_elements, seen_run_elements)
|
||||
for nested_table in cell.tables:
|
||||
self._collect_from_table(nested_table, text_elements)
|
||||
self._collect_from_table(nested_table, text_elements, seen_run_elements)
|
||||
|
||||
def _collect_from_section(
|
||||
self, section: Section, text_elements: List[Tuple[str, Callable[[str], None]]]
|
||||
|
||||
Reference in New Issue
Block a user