sepehr
c794eff823
Some checks failed
Deploy to Production / Build and Deploy (push) Has been cancelled
234 lines
7.0 KiB
Python
234 lines
7.0 KiB
Python
"""
|
|
Prometheus metrics middleware for FastAPI.
|
|
|
|
Exposes /metrics endpoint in Prometheus text format.
|
|
Tracks HTTP requests, translations, and file uploads.
|
|
"""
|
|
|
|
import time
|
|
import logging
|
|
from starlette.middleware.base import BaseHTTPMiddleware
|
|
from starlette.requests import Request
|
|
from starlette.responses import Response
|
|
from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ---- Metrics definitions ----
|
|
|
|
http_requests_total = Counter(
|
|
"http_requests_total",
|
|
"Total HTTP requests",
|
|
["method", "path", "status"],
|
|
)
|
|
|
|
translation_total = Counter(
|
|
"translation_total",
|
|
"Total translations processed",
|
|
["provider", "file_type", "status"],
|
|
)
|
|
|
|
translation_duration_seconds = Histogram(
|
|
"translation_duration_seconds",
|
|
"Translation processing duration in seconds",
|
|
["provider", "file_type"],
|
|
buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300),
|
|
)
|
|
|
|
file_size_bytes = Histogram(
|
|
"file_size_bytes",
|
|
"Uploaded file size in bytes",
|
|
["file_type"],
|
|
buckets=(100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 25_000_000, 50_000_000),
|
|
)
|
|
|
|
# ---- Quality pipeline metrics (L0 + L1) ----
|
|
|
|
quality_l0_checks_total = Counter(
|
|
"quality_l0_checks_total",
|
|
"Total L0 (script+length+pattern) quality checks",
|
|
["result", "file_type"], # result: pass | fail | error
|
|
)
|
|
|
|
quality_l1_judge_total = Counter(
|
|
"quality_l1_judge_total",
|
|
"Total L1 (LLM judge) verdicts",
|
|
["verdict", "model"], # verdict: pass | fail | skip | error
|
|
)
|
|
|
|
quality_l1_judge_duration_seconds = Histogram(
|
|
"quality_l1_judge_duration_seconds",
|
|
"L1 LLM judge call duration in seconds",
|
|
["model"],
|
|
buckets=(0.1, 0.25, 0.5, 1, 2, 5, 10, 30),
|
|
)
|
|
|
|
quality_l1_judge_cost_usd = Histogram(
|
|
"quality_l1_judge_cost_usd",
|
|
"L1 LLM judge estimated cost in USD",
|
|
["model"],
|
|
buckets=(0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1),
|
|
)
|
|
|
|
# ---- L2 Pro premium judge (Track A4) ----
|
|
|
|
quality_l2_judge_total = Counter(
|
|
"quality_l2_judge_total",
|
|
"Total L2 (Pro premium judge, 8-dim) verdicts",
|
|
["verdict", "model", "tier"], # verdict: pass | fail | skip | error
|
|
)
|
|
|
|
quality_l2_judge_duration_seconds = Histogram(
|
|
"quality_l2_judge_duration_seconds",
|
|
"L2 Pro judge call duration in seconds",
|
|
["model"],
|
|
buckets=(0.5, 1, 2, 5, 10, 20, 30, 60),
|
|
)
|
|
|
|
quality_l2_judge_cost_usd = Histogram(
|
|
"quality_l2_judge_cost_usd",
|
|
"L2 Pro judge estimated cost in USD",
|
|
["model"],
|
|
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0),
|
|
)
|
|
|
|
# ---- Retry metrics ----
|
|
|
|
translation_retry_total = Counter(
|
|
"translation_retry_total",
|
|
"Total translation retries triggered by quality issues",
|
|
["reason", "tier"], # reason: l0_fail | l1_fail | format_loss
|
|
)
|
|
|
|
# ---- Format preservation metrics ----
|
|
|
|
format_elements_lost_total = Counter(
|
|
"format_elements_lost_total",
|
|
"Format elements (hyperlinks, footnotes, charts, ...) lost during translation",
|
|
["format", "element_type"], # format: docx | xlsx | pptx | pdf
|
|
)
|
|
|
|
# Paths to skip from metrics (noisy health checks)
|
|
_SKIP_PATHS = {"/health", "/ready", "/metrics", "/favicon.ico"}
|
|
|
|
|
|
def record_translation(provider: str, file_type: str, duration: float, status: str = "success"):
|
|
translation_total.labels(provider=provider, file_type=file_type, status=status).inc()
|
|
translation_duration_seconds.labels(provider=provider, file_type=file_type).observe(duration)
|
|
|
|
|
|
def record_file_size(file_type: str, size_bytes: int):
|
|
file_size_bytes.labels(file_type=file_type).observe(size_bytes)
|
|
|
|
|
|
# ---- Quality helpers ----
|
|
|
|
def record_l0_result(passed: bool, file_type: str = "unknown"):
|
|
"""Record an L0 quality check outcome.
|
|
|
|
Args:
|
|
passed: True if the L0 check passed.
|
|
file_type: docx | xlsx | pptx | pdf | unknown
|
|
"""
|
|
result = "pass" if passed else "fail"
|
|
quality_l0_checks_total.labels(result=result, file_type=file_type).inc()
|
|
|
|
|
|
def record_l0_error(file_type: str = "unknown"):
|
|
"""Record an L0 internal error (so we can spot broken checks)."""
|
|
quality_l0_checks_total.labels(result="error", file_type=file_type).inc()
|
|
|
|
|
|
def record_l1_verdict(
|
|
verdict: str,
|
|
model: str = "unknown",
|
|
duration_seconds: float = None,
|
|
cost_usd: float = None,
|
|
):
|
|
"""Record an L1 judge verdict.
|
|
|
|
Args:
|
|
verdict: pass | fail | skip | error
|
|
model: model name (e.g. deepseek-chat)
|
|
duration_seconds: optional, observed in histogram
|
|
cost_usd: optional, observed in histogram
|
|
"""
|
|
quality_l1_judge_total.labels(verdict=verdict, model=model).inc()
|
|
if duration_seconds is not None:
|
|
quality_l1_judge_duration_seconds.labels(model=model).observe(duration_seconds)
|
|
if cost_usd is not None:
|
|
quality_l1_judge_cost_usd.labels(model=model).observe(cost_usd)
|
|
|
|
|
|
def record_l2_verdict(
|
|
verdict: str,
|
|
model: str = "unknown",
|
|
tier: str = "pro",
|
|
duration_seconds: float = None,
|
|
cost_usd: float = None,
|
|
):
|
|
"""Record an L2 Pro judge verdict.
|
|
|
|
Args:
|
|
verdict: pass | fail | skip | error
|
|
model: model name (e.g. gpt-4o)
|
|
tier: pro | business | enterprise
|
|
duration_seconds: optional, observed in histogram
|
|
cost_usd: optional, observed in histogram
|
|
"""
|
|
quality_l2_judge_total.labels(verdict=verdict, model=model, tier=tier).inc()
|
|
if duration_seconds is not None:
|
|
quality_l2_judge_duration_seconds.labels(model=model).observe(duration_seconds)
|
|
if cost_usd is not None:
|
|
quality_l2_judge_cost_usd.labels(model=model).observe(cost_usd)
|
|
|
|
|
|
def record_translation_retry(reason: str, tier: str = "free"):
|
|
"""Record a translation retry triggered by a quality issue.
|
|
|
|
Args:
|
|
reason: l0_fail | l1_fail | format_loss | user_request
|
|
tier: free | pro | enterprise
|
|
"""
|
|
translation_retry_total.labels(reason=reason, tier=tier).inc()
|
|
|
|
|
|
def record_format_loss(format: str, element_type: str):
|
|
"""Record a format element that was lost during translation.
|
|
|
|
Args:
|
|
format: docx | xlsx | pptx | pdf
|
|
element_type: hyperlink | footnote | chart | diagram | comment | image
|
|
"""
|
|
format_elements_lost_total.labels(format=format, element_type=element_type).inc()
|
|
|
|
|
|
class PrometheusMiddleware(BaseHTTPMiddleware):
|
|
async def dispatch(self, request: Request, call_next):
|
|
if request.url.path in _SKIP_PATHS:
|
|
return await call_next(request)
|
|
|
|
start = time.time()
|
|
response: Response = await call_next(request)
|
|
duration = time.time() - start
|
|
|
|
path = request.url.path
|
|
# Group dynamic paths to avoid label explosion
|
|
if path.startswith("/api/v1/translations/"):
|
|
path = "/api/v1/translations/{id}"
|
|
elif path.startswith("/api/v1/download/"):
|
|
path = "/api/v1/download/{id}"
|
|
|
|
http_requests_total.labels(
|
|
method=request.method,
|
|
path=path,
|
|
status=str(response.status_code),
|
|
).inc()
|
|
|
|
return response
|
|
|
|
|
|
def get_metrics() -> Response:
|
|
body = generate_latest()
|
|
return Response(content=body, media_type=CONTENT_TYPE_LATEST)
|