feat(quality): add L0 quality layer (Track A1 + A2 of dev plan)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 3m5s
L0 quality detection layer to catch translation failures BEFORE they
reach users. Pure Python/TypeScript, zero new dependencies, no API calls.
Backend (Python — services/quality/):
- Script detection: 145 langs mapped to 23 scripts (Latin, Cyrillic,
Greek, Arabic, Hebrew, CJK, Hangul, Kana, Devanagari, Bengali, etc.)
- Language confusion detection (e.g. Arabic text for French target)
- Arabic-script variant discrimination (Persian/Urdu/Pashto/Kurdish
confusion — e.g. Persian text returned when Arabic was requested)
- Length sanity check (with numeric/short-source exemptions)
- Prompt leak detection (Translation: / Voici la traduction: / 翻译:)
- Repetition hallucination detection (token + character level)
- File text extraction for .docx/.xlsx/.pptx/.pdf (no translator
changes needed)
- Defensive pipeline that never raises (L0 must NEVER break a job)
Frontend (TypeScript — wordly.art---traduction-de-documents/src/utils/):
- Exact 1:1 mirror of the Python module
- Zero dependencies, works in browser AND Node.js
- Native Unicode regex (\\p{L}/u) and codePoint iteration
- 63 tests using Node's built-in test runner
Integration:
- Feature-flagged: QUALITY_L0_ENABLED=false (default)
- Observation only: logs structured events, never modifies files
- try/except wrapped: impossible to break a translation job
- Lazy imports: only loaded when flag is on
- Zero impact on existing tests / behavior
Tests:
- 111 Python tests covering all paths (config, script, length, leak,
pipeline, file_extractor) — 100% pass
- 63 TypeScript tests (Node --test) — 100% pass
- 174/174 total tests for the L0 layer
Bug fixes in script mapping:
- yi (Yiddish) -> hebrew (was incorrectly mapped to arabic)
- dv (Maldivian) -> thaana (was incorrectly mapped to arabic)
- ja (Japanese) -> hiragana_katakana (distinguishes from Chinese CJK)
Phase 1 (backend) + Phase 2 (frontend) of Track A complete.
Next: Track B1 (Word/Excel format preservation quick wins).
Closes Track A phase 1+2 of the dev plan.
This commit is contained in:
146
services/quality/file_extractor.py
Normal file
146
services/quality/file_extractor.py
Normal file
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
File text extractor for the L0 quality layer.
|
||||
|
||||
Extracts a small sample of text from a translated file so the L0 checks
|
||||
can run on a real output without requiring the translators to expose
|
||||
their internal chunk data.
|
||||
|
||||
This module depends on the same libraries the translators use
|
||||
(python-docx, openpyxl, python-pptx, PyMuPDF). All imports are lazy
|
||||
and guarded, so a missing library only blocks the matching format —
|
||||
other formats keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, TypedDict
|
||||
|
||||
|
||||
class TextSample(TypedDict):
|
||||
"""A (source_placeholder, translated) pair from an output file."""
|
||||
source: str
|
||||
translated: str
|
||||
|
||||
|
||||
# Maximum samples per format — keeps the L0 check fast.
|
||||
DEFAULT_MAX_SAMPLES = 20
|
||||
|
||||
|
||||
def extract_sample(
|
||||
file_path: Path,
|
||||
file_extension: str,
|
||||
max_samples: int = DEFAULT_MAX_SAMPLES,
|
||||
) -> List[TextSample]:
|
||||
"""
|
||||
Extract a sample of translated text strings from a finished file.
|
||||
|
||||
The "source" field is always empty — we don't have the original
|
||||
document at this point. The L0 checks that care about source/target
|
||||
ratio (length_checker) handle empty source gracefully.
|
||||
|
||||
Returns an empty list if the file cannot be read.
|
||||
"""
|
||||
if not file_path or not Path(file_path).exists():
|
||||
return []
|
||||
|
||||
ext = (file_extension or Path(file_path).suffix).lower()
|
||||
try:
|
||||
if ext == ".docx":
|
||||
return _extract_docx(file_path, max_samples)
|
||||
if ext == ".xlsx":
|
||||
return _extract_xlsx(file_path, max_samples)
|
||||
if ext == ".pptx":
|
||||
return _extract_pptx(file_path, max_samples)
|
||||
if ext == ".pdf":
|
||||
return _extract_pdf(file_path, max_samples)
|
||||
except Exception:
|
||||
# Any failure: return an empty list. The route will log it.
|
||||
return []
|
||||
return []
|
||||
|
||||
|
||||
def _extract_docx(path: Path, max_samples: int) -> List[TextSample]:
|
||||
"""Extract text from a Word document."""
|
||||
from docx import Document
|
||||
doc = Document(str(path))
|
||||
samples: List[TextSample] = []
|
||||
for para in doc.paragraphs:
|
||||
text = (para.text or "").strip()
|
||||
if text and len(text) > 5:
|
||||
samples.append({"source": "", "translated": text})
|
||||
if len(samples) >= max_samples:
|
||||
break
|
||||
# If body had nothing, try tables.
|
||||
if not samples:
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
text = (cell.text or "").strip()
|
||||
if text and len(text) > 5:
|
||||
samples.append({"source": "", "translated": text})
|
||||
if len(samples) >= max_samples:
|
||||
return samples
|
||||
return samples
|
||||
|
||||
|
||||
def _extract_xlsx(path: Path, max_samples: int) -> List[TextSample]:
|
||||
"""Extract text from an Excel file."""
|
||||
from openpyxl import load_workbook
|
||||
wb = load_workbook(str(path), data_only=True, read_only=True)
|
||||
samples: List[TextSample] = []
|
||||
try:
|
||||
for ws in wb.worksheets:
|
||||
for row in ws.iter_rows():
|
||||
for cell in row:
|
||||
val = cell.value
|
||||
if isinstance(val, str):
|
||||
text = val.strip()
|
||||
if text and len(text) > 3:
|
||||
samples.append({"source": "", "translated": text})
|
||||
if len(samples) >= max_samples:
|
||||
return samples
|
||||
finally:
|
||||
wb.close()
|
||||
return samples
|
||||
|
||||
|
||||
def _extract_pptx(path: Path, max_samples: int) -> List[TextSample]:
|
||||
"""Extract text from a PowerPoint file."""
|
||||
from pptx import Presentation
|
||||
pres = Presentation(str(path))
|
||||
samples: List[TextSample] = []
|
||||
for slide in pres.slides:
|
||||
for shape in slide.shapes:
|
||||
if not shape.has_text_frame:
|
||||
continue
|
||||
for para in shape.text_frame.paragraphs:
|
||||
text = (para.text or "").strip()
|
||||
if text and len(text) > 3:
|
||||
samples.append({"source": "", "translated": text})
|
||||
if len(samples) >= max_samples:
|
||||
return samples
|
||||
return samples
|
||||
|
||||
|
||||
def _extract_pdf(path: Path, max_samples: int) -> List[TextSample]:
|
||||
"""Extract text from a PDF file (best-effort, layout-preserving format)."""
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
except ImportError:
|
||||
return []
|
||||
samples: List[TextSample] = []
|
||||
doc = fitz.open(str(path))
|
||||
try:
|
||||
for page in doc:
|
||||
text = page.get_text("text") or ""
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line and len(line) > 5:
|
||||
samples.append({"source": "", "translated": line})
|
||||
if len(samples) >= max_samples:
|
||||
return samples
|
||||
finally:
|
||||
doc.close()
|
||||
return samples
|
||||
Reference in New Issue
Block a user