feat(translation): quality pipeline overhaul + new features (audit 2026-08-29)
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m20s

Translation quality & format preservation:
- Word: merge adjacent same-format runs into one unit (sentence-level
  coherence like inline-tag handling); translate comments/balloons;
  dedupe textbox collection (was translated twice); RTL no longer
  overrides center/justify alignment; CJK/Arabic font hints (eastAsia/cs)
- PPTX: chart translations now actually reach the output file
  (ChartPart.blob is read-only — rewrite chart XML in the saved ZIP);
  CJK typeface hints (a:ea)
- Excel: sheet renames no longer break references — rewrite cell
  formulas (3D/quoted), defined names, data validations, cond. formats
- PDF: bold/italic honored (hebo/heit/hebi); table cells never merge;
  unchanged blocks left untouched (typography preserved, fixes duplicate
  hyperlinks); attempted/changed stats + route gate now cover PDF;
  CJK font paths; scanned PDFs via Mistral OCR (detection + admin settings)

Features:
- formality param (formal/informal) + automatic regional-variant prompts
- output_mode=bilingual docx (source above translation)
- per-user translation memory on Redis (falls back to LRU), context-hashed
- QA report + 0-100 confidence score in job status; L0 on by default
- OpenAI-compatible providers: whole chunk in ONE numbered-JSON request
  (~15x fewer calls) with per-item fallback; base prompt always present
  (custom prompt no longer replaces translation instructions)

Infra & marketing alignment:
- plan-based engine gating + vision gating (closes paid-engine leak);
  /providers/available filtered per plan; 107 languages exposed
- zh-CN/zh-TW validation fixed; libmagic disabled on Windows (native crash)
- admin: Mistral OCR settings + engine status dashboard; httpx<0.28 pin
  (TestClient breakage); Prometheus test fixture fixed
- marketing docs aligned with code (PDF+OCR, retention, engines, pricing)
- security: .env.ionos/.env.production/provider_settings.json removed

Tests: 1173 passed / 0 failed (6 network tests deselected: free Google
endpoint temporarily blocked from this machine)
This commit is contained in:
2026-08-29 18:38:09 +02:00
parent 992f13d53c
commit 526c87348f
87 changed files with 6996 additions and 1024 deletions

View File

@@ -35,36 +35,6 @@ from prometheus_client import (
_REPO_ROOT = Path(__file__).resolve().parent.parent
def _load_metrics_module_with_registry(registry: CollectorRegistry):
"""Load middleware/metrics.py with patched Counter/Histogram to
use the supplied registry. Returns the loaded module."""
spec = importlib.util.spec_from_file_location(
"metrics_under_test",
_REPO_ROOT / "middleware" / "metrics.py",
)
mod = importlib.util.module_from_spec(spec)
# Inject the fresh registry into the module's namespace before exec
mod.__dict__["_TEST_REGISTRY"] = registry
# Patch Counter/Histogram to use the fresh registry
orig_counter = Counter
orig_histogram = Histogram
def _counter(*args, **kwargs):
kwargs.setdefault("registry", registry)
return orig_counter(*args, **kwargs)
def _histogram(*args, **kwargs):
kwargs.setdefault("registry", registry)
return orig_histogram(*args, **kwargs)
mod.__dict__["Counter"] = _counter
mod.__dict__["Histogram"] = _histogram
spec.loader.exec_module(mod)
return mod
@pytest.fixture(scope="module")
def metrics():
"""Load the metrics module ONCE per test module.
@@ -80,55 +50,59 @@ def metrics():
return _load_metrics_module_with_fresh_registry()
def _load_metrics_module_with_registry(registry: CollectorRegistry):
"""Load middleware/metrics.py with ALL its counters/histograms
registered on the supplied fresh registry.
metrics.py does ``from prometheus_client import Counter, Histogram`` at
module top — injecting patched classes into the module dict BEFORE exec
does not survive that import. The only reliable interception point is
the ``prometheus_client`` module itself: we swap its Counter/Histogram
attributes for subclasses that default to ``registry``, exec the
module source, and restore the originals.
"""
import types
import prometheus_client
orig_counter = prometheus_client.Counter
orig_histogram = prometheus_client.Histogram
class _RegistryCounter(orig_counter):
def __new__(cls, *args, **kwargs):
kwargs.setdefault("registry", registry)
return super().__new__(cls)
def __init__(self, *args, **kwargs):
kwargs.setdefault("registry", registry)
super().__init__(*args, **kwargs)
class _RegistryHistogram(orig_histogram):
def __new__(cls, *args, **kwargs):
kwargs.setdefault("registry", registry)
return super().__new__(cls)
def __init__(self, *args, **kwargs):
kwargs.setdefault("registry", registry)
super().__init__(*args, **kwargs)
source = (_REPO_ROOT / "middleware" / "metrics.py").read_text(encoding="utf-8")
mod = types.ModuleType("metrics_under_test")
mod.__file__ = str(_REPO_ROOT / "middleware" / "metrics.py")
try:
prometheus_client.Counter = _RegistryCounter
prometheus_client.Histogram = _RegistryHistogram
exec(compile(source, mod.__file__, "exec"), mod.__dict__)
finally:
prometheus_client.Counter = orig_counter
prometheus_client.Histogram = orig_histogram
return mod
def _load_metrics_module_with_fresh_registry():
"""Load metrics module with its counters/histograms attached to a
fresh CollectorRegistry."""
spec = importlib.util.spec_from_file_location(
"metrics_under_test",
_REPO_ROOT / "middleware" / "metrics.py",
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
# The module just created its counters on the default REGISTRY.
# Unregister them, then re-create them on a fresh registry.
fresh = CollectorRegistry()
for name in (
"http_requests_total",
"translation_total",
"translation_duration_seconds",
"file_size_bytes",
"quality_l0_checks_total",
"quality_l1_judge_total",
"quality_l1_judge_duration_seconds",
"quality_l1_judge_cost_usd",
"translation_retry_total",
"format_elements_lost_total",
):
if not hasattr(mod, name):
continue
obj = getattr(mod, name)
try:
REGISTRY.unregister(obj)
except KeyError:
pass
# Now re-import the module so the new metrics register on `fresh`
spec = importlib.util.spec_from_file_location(
"metrics_under_test_isolated",
_REPO_ROOT / "middleware" / "metrics.py",
)
mod2 = importlib.util.module_from_spec(spec)
# We can't easily re-route Counter/Histogram in exec_module because
# they call into the global REGISTRY via the function signature.
# Instead: reload by re-importing via importlib with a wrapper
# that intercepts the Counter/Histogram constructors. We do this
# via the more direct route: use the EXISTING counters on the
# default REGISTRY, but only check RELATIVE increments.
#
# Practical approach: just use the module as-is. Tests check
# `after >= before + 1`, which is robust against other tests.
return mod
fresh CollectorRegistry (never the shared default one — the app may
already have registered the same names earlier in the session)."""
return _load_metrics_module_with_registry(CollectorRegistry())
def _counter_value(counter, **labels):