feat(monitoring): observability stack + app instrumentation
- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
# Alert rules surfaced in Grafana's alerting UI (no Alertmanager needed at
|
||||
# this scale — Grafana can route them later via email/Telegram).
|
||||
groups:
|
||||
- name: erp-availability
|
||||
rules:
|
||||
- alert: ErpApiDown
|
||||
expr: up{job="erp-api"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ERP API is not scraping (down or /metrics failing)"
|
||||
|
||||
- alert: ErpApiHighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{job="erp-api", status=~"5.."}[5m]))
|
||||
/ clamp_min(sum(rate(http_requests_total{job="erp-api"}[5m])), 0.001) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "ERP API 5xx rate above 5% for 5 minutes"
|
||||
|
||||
- alert: ErpApiSlowRequests
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(http_request_duration_seconds_bucket{job="erp-api"}[5m])) by (le)) > 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "ERP API p95 latency above 2s for 10 minutes"
|
||||
|
||||
- name: host
|
||||
rules:
|
||||
- alert: HostDiskAlmostFull
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"}) < 0.1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Root filesystem below 10% free (deploy builds will fail)"
|
||||
|
||||
- alert: HostMemoryPressure
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host RAM below 10% available (check monitoring stack limits)"
|
||||
|
||||
- alert: PostgresTooManyConnections
|
||||
expr: sum(pg_stat_activity_count) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PostgreSQL connection count above 80"
|
||||
@@ -0,0 +1,34 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
# The ERP API — /metrics is exposed by prometheus-fastapi-instrumentator
|
||||
# (request rate, latency histogram, error counts). Reached over the
|
||||
# erp_shared network the main compose also joins.
|
||||
- job_name: erp-api
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ["api:8000"]
|
||||
|
||||
# Host-level metrics (CPU, RAM, disk, network).
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
|
||||
# Per-container metrics (CPU, memory, restarts) for every container.
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ["cadvisor:8080"]
|
||||
|
||||
# App database metrics (connections, cache hit ratio, WAL, ...).
|
||||
- job_name: postgres
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
Reference in New Issue
Block a user