"""Bot observability metrics (read-only) for the ERP admin dashboard page. Reads the bot's own tables (bot_traces / bot_trace_events — owned and written by the Bale bot container) with plain SELECTs only. Admin-gated. Calibration, demo and test traffic (source != 'bale' / chat_id = 0) is excluded from the aggregates; the traces endpoint can show everything with ?include_synthetic=1. """ from fastapi import APIRouter, Depends, Query from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession from app.auth import require_role from app.db.engine import get_db router = APIRouter(prefix="/bot-metrics", tags=["bot-metrics"]) # real-traffic predicate shared by every aggregate (same rule as the bot's # tools/monitor.py so the numbers match the weekly digest) REAL = ("e.trace_id IN (SELECT id FROM bot_traces" " WHERE coalesce(source,'bale') NOT IN ('calibration','demo')" " AND coalesce(chat_id,0) <> 0)") async def _rows(db: AsyncSession, sql: str, params: dict | None = None) -> list[dict]: res = await db.execute(text(sql), params or {}) return [dict(r._mapping) for r in res.all()] @router.get("/summary") async def summary( days: int = Query(7, ge=1, le=90), db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """Headline KPIs + daily volumes for the dashboard header cards.""" p = {"days": days} kpis = (await _rows(db, f""" SELECT (SELECT count(*) FROM bot_traces WHERE coalesce(source,'bale') NOT IN ('calibration','demo') AND created_at >= now() - (:days * interval '1 day')) AS traces, (SELECT count(*) FROM bot_trace_events e WHERE {REAL} AND e.created_at >= now() - (:days * interval '1 day')) AS events, (SELECT count(*) FROM bot_trace_events e WHERE {REAL} AND (e.stage = 'jev.failure' OR (e.stage = 'jev.result' AND NOT e.ok) OR e.stage LIKE 'matching.%.failure') AND e.created_at >= now() - (:days * interval '1 day')) AS errors, (SELECT coalesce(round(percentile_cont(0.95) WITHIN GROUP (ORDER BY e.latency_ms)), 0) FROM bot_trace_events e WHERE {REAL} AND e.stage = 'jev.result' AND e.latency_ms IS NOT NULL AND e.created_at >= now() - (:days * interval '1 day')) AS jev_p95_ms, (SELECT count(*) FROM bot_trace_events e WHERE {REAL} AND e.stage = 'draft.commit' AND e.created_at >= now() - (:days * interval '1 day')) AS commits, (SELECT coalesce(sum((e.payload->'usage'->>'input_tokens')::bigint), 0) FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result') AND e.payload->'usage' IS NOT NULL AND e.created_at >= now() - (:days * interval '1 day')) AS tokens_in, (SELECT coalesce(sum((e.payload->'usage'->>'output_tokens')::bigint), 0) FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result') AND e.payload->'usage' IS NOT NULL AND e.created_at >= now() - (:days * interval '1 day')) AS tokens_out """, p))[0] daily = await _rows(db, f""" SELECT to_char(date_trunc('day', e.created_at), 'YYYY-MM-DD') AS day, count(*) AS events, count(DISTINCT e.trace_id) AS traces FROM bot_trace_events e WHERE {REAL} AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY 1 """, p) return {"data": {"kpis": kpis, "daily": daily, "days": days}} @router.get("/routing") async def routing( days: int = Query(7, ge=1, le=90), db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """Final mention statuses (arbiter + crisis fallback aggregates, plus the legacy per-mention rows) and the arbiter's actions on Jev's suggestions.""" p = {"days": days} statuses = await _rows(db, f""" SELECT metric, sum(n)::int AS n FROM ( SELECT unnest(ARRAY['exact','ambiguity','new']) AS metric, 0 AS n UNION ALL SELECT 'exact', sum(coalesce((e.payload->'statuses'->>'exact')::int,0)) FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('matching.arbitration.result','matching.fallback.result') AND e.created_at >= now() - (:days * interval '1 day') UNION ALL SELECT 'ambiguity', sum(coalesce((e.payload->'statuses'->>'ambiguity')::int,0)) FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('matching.arbitration.result','matching.fallback.result') AND e.created_at >= now() - (:days * interval '1 day') UNION ALL SELECT 'new', sum(coalesce((e.payload->'statuses'->>'new')::int,0)) FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('matching.arbitration.result','matching.fallback.result') AND e.created_at >= now() - (:days * interval '1 day') UNION ALL SELECT m->>'status', count(*) FROM bot_trace_events e JOIN bot_traces t ON t.id = e.trace_id, jsonb_array_elements(e.payload->'mentions') m WHERE e.stage='routing.decision' AND e.payload->>'source'='jev' AND coalesce(t.source,'bale') NOT IN ('calibration','demo') AND coalesce(t.chat_id,0) <> 0 AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ) x GROUP BY metric """, p) actions = await _rows(db, f""" SELECT coalesce(sum((e.payload->'actions'->>'endorsed_jev')::int),0)::int AS endorsed_jev, coalesce(sum((e.payload->'actions'->>'vetoed_to_new')::int),0)::int AS vetoed_to_new, coalesce(sum((e.payload->'actions'->>'demoted_to_ambiguity')::int),0)::int AS demoted_to_ambiguity, coalesce(sum((e.payload->'actions'->>'other')::int),0)::int AS other, count(*)::int AS invocations FROM bot_trace_events e WHERE {REAL} AND e.stage='matching.arbitration.result' AND e.created_at >= now() - (:days * interval '1 day') """, p) stages = await _rows(db, f""" SELECT e.stage, count(*)::int AS n FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('jev.request','jev.result','jev.failure', 'matching.arbitration.result','matching.fallback.result') AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY n DESC """, p) return {"data": {"statuses": statuses, "actions": actions[0] if actions else {}, "stages": stages}} @router.get("/quality") async def quality( days: int = Query(7, ge=1, le=90), db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """User choices as the real A/B proxy: what the user picked when asked.""" p = {"days": days} amb = await _rows(db, f""" SELECT coalesce(e.payload->>'action','?') AS action, coalesce(e.payload->>'arg','-') AS arg, count(*)::int AS n FROM bot_trace_events e WHERE {REAL} AND e.stage='ambiguity.user_choice' AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1,2 ORDER BY n DESC """, p) picks = await _rows(db, f""" SELECT CASE WHEN (e.payload->>'idx')::int = 0 THEN 'top' ELSE 'other' END AS pick, count(*)::int AS n FROM bot_trace_events e WHERE {REAL} AND e.stage='user_choice.pick' AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 """, p) drafts = await _rows(db, f""" SELECT e.stage, count(*)::int AS n FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('draft.render','draft.commit','draft.cancel','ambiguity.sent') AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY n DESC """, p) return {"data": {"ambiguities": amb, "picks": picks, "drafts": drafts}} @router.get("/jev") async def jev_health( days: int = Query(7, ge=1, le=90), db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """Jev health: models seen (drift vs the pinned version), tokens, latency.""" p = {"days": days} models = await _rows(db, f""" SELECT coalesce(e.model,'?') AS model, count(*)::int AS n, max(e.created_at)::text AS last_seen FROM bot_trace_events e WHERE {REAL} AND e.stage='jev.result' AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY n DESC """, p) latency = await _rows(db, f""" SELECT e.stage, coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50, coalesce(round(percentile_cont(0.95) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p95, count(*)::int AS n FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('jev.result','nlu.result','matching.arbitration.result','matching.fallback.result') AND e.latency_ms IS NOT NULL AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY n DESC """, p) tokens_daily = await _rows(db, f""" SELECT to_char(date_trunc('day', e.created_at), 'YYYY-MM-DD') AS day, coalesce(sum((e.payload->'usage'->>'input_tokens')::bigint),0)::bigint AS tokens_in, coalesce(sum((e.payload->'usage'->>'output_tokens')::bigint),0)::bigint AS tokens_out FROM bot_trace_events e WHERE {REAL} AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result') AND e.payload->'usage' IS NOT NULL AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY 1 """, p) return {"data": {"models": models, "latency": latency, "tokens_daily": tokens_daily}} @router.get("/stt") async def stt_health( days: int = Query(7, ge=1, le=90), db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """STT providers and the audio-enhancement layer (ok-rate + p50 latency).""" p = {"days": days} providers = await _rows(db, f""" SELECT coalesce(e.payload->>'provider','?') AS provider, count(*) FILTER (WHERE e.ok)::int AS ok, count(*)::int AS total, coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50_ms FROM bot_trace_events e WHERE {REAL} AND e.stage='stt.result' AND e.created_at >= now() - (:days * interval '1 day') GROUP BY 1 ORDER BY ok DESC """, p) enhance = await _rows(db, f""" SELECT count(*) FILTER (WHERE e.ok)::int AS ok, count(*)::int AS total, coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50_ms FROM bot_trace_events e WHERE {REAL} AND e.stage='audio.enhance.result' AND e.created_at >= now() - (:days * interval '1 day') """, p) return {"data": {"providers": providers, "enhance": enhance[0] if enhance else None}} @router.get("/traces") async def traces( limit: int = Query(50, ge=1, le=200), status: str | None = None, include_synthetic: bool = False, db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """Latest traces for the explorer table (oldest events come from /traces/{id}).""" sql = """ SELECT t.id::text AS id, t.created_at::text AS created_at, coalesce(t.source,'bale') AS source, t.chat_id::text AS chat_id, t.kind, coalesce(t.status,'open') AS status, (SELECT count(*) FROM bot_trace_events e WHERE e.trace_id=t.id)::int AS events FROM bot_traces t WHERE (:include_synthetic OR (coalesce(t.source,'bale') NOT IN ('calibration','demo') AND coalesce(t.chat_id,0) <> 0)) """ params: dict = {"include_synthetic": include_synthetic} if status: sql += " AND t.status = :status" params["status"] = status sql += " ORDER BY t.created_at DESC LIMIT :limit" params["limit"] = limit return {"data": await _rows(db, sql, params)} @router.get("/traces/{trace_id}") async def trace_detail( trace_id: str, db: AsyncSession = Depends(get_db), _ctx=Depends(require_role("admin")), ): """Full event list of one trace (the drawer/drill-down view).""" events = await _rows(db, """ SELECT e.created_at::text AS created_at, e.stage, e.ok, e.latency_ms, e.model, e.field, e.payload FROM bot_trace_events e WHERE e.trace_id::text = :tid ORDER BY e.created_at ASC, e.id ASC LIMIT 500 """, {"tid": trace_id}) head = await _rows(db, """ SELECT id::text AS id, created_at::text AS created_at, coalesce(source,'bale') AS source, chat_id::text AS chat_id, kind, coalesce(status,'open') AS status, draft_id::text AS draft_id, committed_quote_id::text AS committed_quote_id FROM bot_traces WHERE id::text = :tid """, {"tid": trace_id}) return {"data": {"trace": head[0] if head else None, "events": events}}