MMD
e17d1f0883
- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
59 lines
1.9 KiB
YAML
59 lines
1.9 KiB
YAML
# Alert rules surfaced in Grafana's alerting UI (no Alertmanager needed at
|
|
# this scale — Grafana can route them later via email/Telegram).
|
|
groups:
|
|
- name: erp-availability
|
|
rules:
|
|
- alert: ErpApiDown
|
|
expr: up{job="erp-api"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "ERP API is not scraping (down or /metrics failing)"
|
|
|
|
- alert: ErpApiHighErrorRate
|
|
expr: |
|
|
sum(rate(http_requests_total{job="erp-api", status=~"5.."}[5m]))
|
|
/ clamp_min(sum(rate(http_requests_total{job="erp-api"}[5m])), 0.001) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "ERP API 5xx rate above 5% for 5 minutes"
|
|
|
|
- alert: ErpApiSlowRequests
|
|
expr: |
|
|
histogram_quantile(0.95,
|
|
sum(rate(http_request_duration_seconds_bucket{job="erp-api"}[5m])) by (le)) > 2
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "ERP API p95 latency above 2s for 10 minutes"
|
|
|
|
- name: host
|
|
rules:
|
|
- alert: HostDiskAlmostFull
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"}) < 0.1
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Root filesystem below 10% free (deploy builds will fail)"
|
|
|
|
- alert: HostMemoryPressure
|
|
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Host RAM below 10% available (check monitoring stack limits)"
|
|
|
|
- alert: PostgresTooManyConnections
|
|
expr: sum(pg_stat_activity_count) > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "PostgreSQL connection count above 80"
|