feat(monitoring): observability stack + app instrumentation
- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
This commit is contained in:
MMD
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"annotations": { "list": [] },
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (status) (rate(http_requests_total{job=\"erp-api\"}[5m]))",
|
||||
"legendFormat": "HTTP {{status}}"
|
||||
}
|
||||
],
|
||||
"title": "API — request rate by status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 10, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 1 },
|
||||
{ "color": "red", "value": 2 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"id": 2,
|
||||
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
],
|
||||
"title": "API — latency (p50 / p95)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"id": 3,
|
||||
"options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (name) (container_memory_working_set_bytes{job=\"cadvisor\", name!=\"\", image!=\"\"})",
|
||||
"legendFormat": "{{name}}"
|
||||
}
|
||||
],
|
||||
"title": "Containers — memory in use",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"id": 4,
|
||||
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum(pg_stat_activity_count)",
|
||||
"legendFormat": "postgres connections"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "host RAM available"
|
||||
}
|
||||
],
|
||||
"title": "Database connections / host RAM",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
|
||||
"id": 5,
|
||||
"options": {
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true,
|
||||
"showTime": true,
|
||||
"enableLogDetails": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"expr": "{compose_project=\"erp\"}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "All ERP container logs (Loki)",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["erp"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"hide": 2,
|
||||
"current": { "text": "Prometheus", "value": "Prometheus" }
|
||||
},
|
||||
{
|
||||
"name": "DS_LOKI",
|
||||
"label": "Loki",
|
||||
"type": "datasource",
|
||||
"query": "loki",
|
||||
"hide": 2,
|
||||
"current": { "text": "Loki", "value": "Loki" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "ERP — Stack overview",
|
||||
"uid": "erp-overview",
|
||||
"version": 1
|
||||
}
|
||||
Reference in New Issue
Block a user