whqreza
0adadd2cf8
Grafana (full use of the existing monitoring/ stack, per OSS policy):
- datasources.yml: BotPostgres (uid bot-postgres) over the shared network;
grafana service joins 'shared' + carries the datasource env
- db_init.py: idempotent read-only role obs_reader (USAGE on schema public +
SELECT on bot_traces/bot_trace_events; skipped when
OBS_READER_PASSWORD empty; ALTER ROLE for rotation) + composite index
ix_bot_trace_events_stage_time (stage, created_at DESC) — senior-engineer
review applied with the column-name fix (event_type -> stage); (trace_id)
already covered by the existing (trace_id, created_at) index
- dashboards/bot-observability.json: KPI row (real traffic, events, Jev/
fallback errors, Jev p95, tokens, commits, archive guard), events/day,
per-stage p50/p95, mention statuses (arbiter+fallback aggregates + legacy
rows), DeepSeek arbiter actions, tokens/day, model-drift vs jev-1.13.0,
user-choice A/B, STT + audio.enhance health, archive size guard
- dashboards/bot-trace-explorer.json: trace table + full event drill-down
via the trace_id variable
ERP page (admin-only, Persian RTL, recharts@^3 for React 19):
- backend bot_metrics.py: read-only require_role('admin') endpoints
(summary/routing/quality/jev/stt/traces + trace detail); aggregates count
real traffic only (calibration/demo/chat_id=0 excluded)
- frontend BotMonitorPage at /bot-monitor (KPI cards, daily events,
status donut, arbiter actions, user-choice A/B, STT/enhance tables,
stage latency, tokens chart, traces table + detail drawer with payload
viewer); entry link from the admin panel; MobileTabBar hidden there
- api.ts: typed botMetrics client section
Tooling/docs:
- tools/seed_observability_demo.py: deterministic demo traces (source='demo',
every current pipeline stage incl. audio.enhance/jev.failure/arbitration/
fallback/user choices) — excluded from real-traffic aggregates
- monitoring/README.md + bot/docs/DEPLOY-AND-MONITORING.md: setup, seed and
rollback notes; alerts deliberately deferred (owner decision 2026-10-24)
Verified locally: obs_reader SELECT ok / INSERT denied; panels' SQL run as
obs_reader with data; datasource health OK; both dashboards survive a grafana
restart; endpoints 200 as admin and 403 without role; tsc + vite build clean;
page renders in-browser (admin session); bot startup provisions role+index.
283 lines
13 KiB
Python
283 lines
13 KiB
Python
"""Bot observability metrics (read-only) for the ERP admin dashboard page.
|
|
|
|
Reads the bot's own tables (bot_traces / bot_trace_events — owned and written
|
|
by the Bale bot container) with plain SELECTs only. Admin-gated. Calibration,
|
|
demo and test traffic (source != 'bale' / chat_id = 0) is excluded from the
|
|
aggregates; the traces endpoint can show everything with ?include_synthetic=1.
|
|
"""
|
|
from fastapi import APIRouter, Depends, Query
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.auth import require_role
|
|
from app.db.engine import get_db
|
|
|
|
router = APIRouter(prefix="/bot-metrics", tags=["bot-metrics"])
|
|
|
|
# real-traffic predicate shared by every aggregate (same rule as the bot's
|
|
# tools/monitor.py so the numbers match the weekly digest)
|
|
REAL = ("e.trace_id IN (SELECT id FROM bot_traces"
|
|
" WHERE coalesce(source,'bale') NOT IN ('calibration','demo')"
|
|
" AND coalesce(chat_id,0) <> 0)")
|
|
|
|
|
|
async def _rows(db: AsyncSession, sql: str, params: dict | None = None) -> list[dict]:
|
|
res = await db.execute(text(sql), params or {})
|
|
return [dict(r._mapping) for r in res.all()]
|
|
|
|
|
|
@router.get("/summary")
|
|
async def summary(
|
|
days: int = Query(7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""Headline KPIs + daily volumes for the dashboard header cards."""
|
|
p = {"days": days}
|
|
kpis = (await _rows(db, f"""
|
|
SELECT
|
|
(SELECT count(*) FROM bot_traces
|
|
WHERE coalesce(source,'bale') NOT IN ('calibration','demo')
|
|
AND created_at >= now() - (:days * interval '1 day')) AS traces,
|
|
(SELECT count(*) FROM bot_trace_events e WHERE {REAL}
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS events,
|
|
(SELECT count(*) FROM bot_trace_events e WHERE {REAL}
|
|
AND (e.stage = 'jev.failure' OR (e.stage = 'jev.result' AND NOT e.ok)
|
|
OR e.stage LIKE 'matching.%.failure')
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS errors,
|
|
(SELECT coalesce(round(percentile_cont(0.95) WITHIN GROUP (ORDER BY e.latency_ms)), 0)
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage = 'jev.result'
|
|
AND e.latency_ms IS NOT NULL
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS jev_p95_ms,
|
|
(SELECT count(*) FROM bot_trace_events e WHERE {REAL} AND e.stage = 'draft.commit'
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS commits,
|
|
(SELECT coalesce(sum((e.payload->'usage'->>'input_tokens')::bigint), 0)
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result')
|
|
AND e.payload->'usage' IS NOT NULL
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS tokens_in,
|
|
(SELECT coalesce(sum((e.payload->'usage'->>'output_tokens')::bigint), 0)
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result')
|
|
AND e.payload->'usage' IS NOT NULL
|
|
AND e.created_at >= now() - (:days * interval '1 day')) AS tokens_out
|
|
""", p))[0]
|
|
daily = await _rows(db, f"""
|
|
SELECT to_char(date_trunc('day', e.created_at), 'YYYY-MM-DD') AS day,
|
|
count(*) AS events,
|
|
count(DISTINCT e.trace_id) AS traces
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY 1
|
|
""", p)
|
|
return {"data": {"kpis": kpis, "daily": daily, "days": days}}
|
|
|
|
|
|
@router.get("/routing")
|
|
async def routing(
|
|
days: int = Query(7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""Final mention statuses (arbiter + crisis fallback aggregates, plus the
|
|
legacy per-mention rows) and the arbiter's actions on Jev's suggestions."""
|
|
p = {"days": days}
|
|
statuses = await _rows(db, f"""
|
|
SELECT metric, sum(n)::int AS n FROM (
|
|
SELECT unnest(ARRAY['exact','ambiguity','new']) AS metric, 0 AS n
|
|
UNION ALL
|
|
SELECT 'exact', sum(coalesce((e.payload->'statuses'->>'exact')::int,0))
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('matching.arbitration.result','matching.fallback.result')
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
UNION ALL
|
|
SELECT 'ambiguity', sum(coalesce((e.payload->'statuses'->>'ambiguity')::int,0))
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('matching.arbitration.result','matching.fallback.result')
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
UNION ALL
|
|
SELECT 'new', sum(coalesce((e.payload->'statuses'->>'new')::int,0))
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('matching.arbitration.result','matching.fallback.result')
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
UNION ALL
|
|
SELECT m->>'status', count(*) FROM bot_trace_events e
|
|
JOIN bot_traces t ON t.id = e.trace_id, jsonb_array_elements(e.payload->'mentions') m
|
|
WHERE e.stage='routing.decision' AND e.payload->>'source'='jev'
|
|
AND coalesce(t.source,'bale') NOT IN ('calibration','demo')
|
|
AND coalesce(t.chat_id,0) <> 0
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1
|
|
) x GROUP BY metric
|
|
""", p)
|
|
actions = await _rows(db, f"""
|
|
SELECT
|
|
coalesce(sum((e.payload->'actions'->>'endorsed_jev')::int),0)::int AS endorsed_jev,
|
|
coalesce(sum((e.payload->'actions'->>'vetoed_to_new')::int),0)::int AS vetoed_to_new,
|
|
coalesce(sum((e.payload->'actions'->>'demoted_to_ambiguity')::int),0)::int AS demoted_to_ambiguity,
|
|
coalesce(sum((e.payload->'actions'->>'other')::int),0)::int AS other,
|
|
count(*)::int AS invocations
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='matching.arbitration.result'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
""", p)
|
|
stages = await _rows(db, f"""
|
|
SELECT e.stage, count(*)::int AS n
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('jev.request','jev.result','jev.failure',
|
|
'matching.arbitration.result','matching.fallback.result')
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY n DESC
|
|
""", p)
|
|
return {"data": {"statuses": statuses, "actions": actions[0] if actions else {}, "stages": stages}}
|
|
|
|
|
|
@router.get("/quality")
|
|
async def quality(
|
|
days: int = Query(7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""User choices as the real A/B proxy: what the user picked when asked."""
|
|
p = {"days": days}
|
|
amb = await _rows(db, f"""
|
|
SELECT coalesce(e.payload->>'action','?') AS action,
|
|
coalesce(e.payload->>'arg','-') AS arg,
|
|
count(*)::int AS n
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='ambiguity.user_choice'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1,2 ORDER BY n DESC
|
|
""", p)
|
|
picks = await _rows(db, f"""
|
|
SELECT CASE WHEN (e.payload->>'idx')::int = 0 THEN 'top' ELSE 'other' END AS pick,
|
|
count(*)::int AS n
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='user_choice.pick'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1
|
|
""", p)
|
|
drafts = await _rows(db, f"""
|
|
SELECT e.stage, count(*)::int AS n
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('draft.render','draft.commit','draft.cancel','ambiguity.sent')
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY n DESC
|
|
""", p)
|
|
return {"data": {"ambiguities": amb, "picks": picks, "drafts": drafts}}
|
|
|
|
|
|
@router.get("/jev")
|
|
async def jev_health(
|
|
days: int = Query(7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""Jev health: models seen (drift vs the pinned version), tokens, latency."""
|
|
p = {"days": days}
|
|
models = await _rows(db, f"""
|
|
SELECT coalesce(e.model,'?') AS model, count(*)::int AS n,
|
|
max(e.created_at)::text AS last_seen
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='jev.result'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY n DESC
|
|
""", p)
|
|
latency = await _rows(db, f"""
|
|
SELECT e.stage,
|
|
coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50,
|
|
coalesce(round(percentile_cont(0.95) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p95,
|
|
count(*)::int AS n
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('jev.result','nlu.result','matching.arbitration.result','matching.fallback.result')
|
|
AND e.latency_ms IS NOT NULL
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY n DESC
|
|
""", p)
|
|
tokens_daily = await _rows(db, f"""
|
|
SELECT to_char(date_trunc('day', e.created_at), 'YYYY-MM-DD') AS day,
|
|
coalesce(sum((e.payload->'usage'->>'input_tokens')::bigint),0)::bigint AS tokens_in,
|
|
coalesce(sum((e.payload->'usage'->>'output_tokens')::bigint),0)::bigint AS tokens_out
|
|
FROM bot_trace_events e WHERE {REAL}
|
|
AND e.stage IN ('jev.result','matching.arbitration.result','matching.fallback.result')
|
|
AND e.payload->'usage' IS NOT NULL
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY 1
|
|
""", p)
|
|
return {"data": {"models": models, "latency": latency, "tokens_daily": tokens_daily}}
|
|
|
|
|
|
@router.get("/stt")
|
|
async def stt_health(
|
|
days: int = Query(7, ge=1, le=90),
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""STT providers and the audio-enhancement layer (ok-rate + p50 latency)."""
|
|
p = {"days": days}
|
|
providers = await _rows(db, f"""
|
|
SELECT coalesce(e.payload->>'provider','?') AS provider,
|
|
count(*) FILTER (WHERE e.ok)::int AS ok,
|
|
count(*)::int AS total,
|
|
coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50_ms
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='stt.result'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
GROUP BY 1 ORDER BY ok DESC
|
|
""", p)
|
|
enhance = await _rows(db, f"""
|
|
SELECT count(*) FILTER (WHERE e.ok)::int AS ok,
|
|
count(*)::int AS total,
|
|
coalesce(round(percentile_cont(0.5) WITHIN GROUP (ORDER BY e.latency_ms)),0)::int AS p50_ms
|
|
FROM bot_trace_events e WHERE {REAL} AND e.stage='audio.enhance.result'
|
|
AND e.created_at >= now() - (:days * interval '1 day')
|
|
""", p)
|
|
return {"data": {"providers": providers, "enhance": enhance[0] if enhance else None}}
|
|
|
|
|
|
@router.get("/traces")
|
|
async def traces(
|
|
limit: int = Query(50, ge=1, le=200),
|
|
status: str | None = None,
|
|
include_synthetic: bool = False,
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""Latest traces for the explorer table (oldest events come from /traces/{id})."""
|
|
sql = """
|
|
SELECT t.id::text AS id, t.created_at::text AS created_at,
|
|
coalesce(t.source,'bale') AS source, t.chat_id::text AS chat_id,
|
|
t.kind, coalesce(t.status,'open') AS status,
|
|
(SELECT count(*) FROM bot_trace_events e WHERE e.trace_id=t.id)::int AS events
|
|
FROM bot_traces t
|
|
WHERE (:include_synthetic
|
|
OR (coalesce(t.source,'bale') NOT IN ('calibration','demo') AND coalesce(t.chat_id,0) <> 0))
|
|
"""
|
|
params: dict = {"include_synthetic": include_synthetic}
|
|
if status:
|
|
sql += " AND t.status = :status"
|
|
params["status"] = status
|
|
sql += " ORDER BY t.created_at DESC LIMIT :limit"
|
|
params["limit"] = limit
|
|
return {"data": await _rows(db, sql, params)}
|
|
|
|
|
|
@router.get("/traces/{trace_id}")
|
|
async def trace_detail(
|
|
trace_id: str,
|
|
db: AsyncSession = Depends(get_db),
|
|
_ctx=Depends(require_role("admin")),
|
|
):
|
|
"""Full event list of one trace (the drawer/drill-down view)."""
|
|
events = await _rows(db, """
|
|
SELECT e.created_at::text AS created_at, e.stage, e.ok,
|
|
e.latency_ms, e.model, e.field, e.payload
|
|
FROM bot_trace_events e
|
|
WHERE e.trace_id::text = :tid
|
|
ORDER BY e.created_at ASC, e.id ASC
|
|
LIMIT 500
|
|
""", {"tid": trace_id})
|
|
head = await _rows(db, """
|
|
SELECT id::text AS id, created_at::text AS created_at,
|
|
coalesce(source,'bale') AS source, chat_id::text AS chat_id,
|
|
kind, coalesce(status,'open') AS status,
|
|
draft_id::text AS draft_id, committed_quote_id::text AS committed_quote_id
|
|
FROM bot_traces WHERE id::text = :tid
|
|
""", {"tid": trace_id})
|
|
return {"data": {"trace": head[0] if head else None, "events": events}}
|