feat(quality): A4 — L2 Pro premium judge (8 dims, gpt-4o, Pro-gated, opt-in)
Some checks failed
Deploy to Production / Build and Deploy (push) Has been cancelled
Some checks failed
Deploy to Production / Build and Deploy (push) Has been cancelled
This commit is contained in:
@@ -70,6 +70,28 @@ quality_l1_judge_cost_usd = Histogram(
|
||||
buckets=(0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1),
|
||||
)
|
||||
|
||||
# ---- L2 Pro premium judge (Track A4) ----
|
||||
|
||||
quality_l2_judge_total = Counter(
|
||||
"quality_l2_judge_total",
|
||||
"Total L2 (Pro premium judge, 8-dim) verdicts",
|
||||
["verdict", "model", "tier"], # verdict: pass | fail | skip | error
|
||||
)
|
||||
|
||||
quality_l2_judge_duration_seconds = Histogram(
|
||||
"quality_l2_judge_duration_seconds",
|
||||
"L2 Pro judge call duration in seconds",
|
||||
["model"],
|
||||
buckets=(0.5, 1, 2, 5, 10, 20, 30, 60),
|
||||
)
|
||||
|
||||
quality_l2_judge_cost_usd = Histogram(
|
||||
"quality_l2_judge_cost_usd",
|
||||
"L2 Pro judge estimated cost in USD",
|
||||
["model"],
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0),
|
||||
)
|
||||
|
||||
# ---- Retry metrics ----
|
||||
|
||||
translation_retry_total = Counter(
|
||||
@@ -138,6 +160,29 @@ def record_l1_verdict(
|
||||
quality_l1_judge_cost_usd.labels(model=model).observe(cost_usd)
|
||||
|
||||
|
||||
def record_l2_verdict(
|
||||
verdict: str,
|
||||
model: str = "unknown",
|
||||
tier: str = "pro",
|
||||
duration_seconds: float = None,
|
||||
cost_usd: float = None,
|
||||
):
|
||||
"""Record an L2 Pro judge verdict.
|
||||
|
||||
Args:
|
||||
verdict: pass | fail | skip | error
|
||||
model: model name (e.g. gpt-4o)
|
||||
tier: pro | business | enterprise
|
||||
duration_seconds: optional, observed in histogram
|
||||
cost_usd: optional, observed in histogram
|
||||
"""
|
||||
quality_l2_judge_total.labels(verdict=verdict, model=model, tier=tier).inc()
|
||||
if duration_seconds is not None:
|
||||
quality_l2_judge_duration_seconds.labels(model=model).observe(duration_seconds)
|
||||
if cost_usd is not None:
|
||||
quality_l2_judge_cost_usd.labels(model=model).observe(cost_usd)
|
||||
|
||||
|
||||
def record_translation_retry(reason: str, tier: str = "free"):
|
||||
"""Record a translation retry triggered by a quality issue.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user