feat(monitoring): observability stack + app instrumentation

- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
2026-09-18 22:18:02 +03:30
parent f7b1f7f751
commit e17d1f0883
26 changed files with 1106 additions and 3 deletions
@@ -0,0 +1,191 @@
+# ============================================================================
+# Monitoring stack — SEPARATE compose project from the ERP app on purpose:
+# app rebuilds (push-to-deploy) must never restart monitoring containers.
+#
+#   cd /srv/erp   # repo root, so --env-file picks up the app's .env
+#   docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d
+#
+# Included (≈2.2 GB RAM budgeted):
+#   GlitchTip (errors, Sentry-compatible) + its own small Postgres
+#   Prometheus (metrics) + Grafana (dashboards) + Loki/Promtail (logs)
+#   node-exporter (host) + postgres-exporter (app DB) + cAdvisor (containers)
+#
+# Admin UIs (Grafana :3001, GlitchTip :8001) bind to 127.0.0.1 only —
+# reach them via SSH tunnel exactly like the MinIO console. GlitchTip ALSO
+# needs a public origin for the browser SDK: set GLITCHTIP_HOST and enable
+# the "public" profile (Traefik route + TLS) — see monitoring/README.md.
+# ============================================================================
+
+services:
+  # ---------------------------------------------------------------- errors --
+  glitchtip:
+    image: glitchtip/glitchtip:latest
+    container_name: monitoring-glitchtip
+    mem_limit: 640m
+    environment:
+      SECRET_KEY: ${GLITCHTIP_SECRET_KEY:?set GLITCHTIP_SECRET_KEY in .env}
+      GLITCHTIP_DOMAIN: ${GLITCHTIP_DOMAIN:?set GLITCHTIP_DOMAIN in .env}
+      DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASSWORD:-glitchtip}@glitchtip-postgres:5432/glitchtip
+      # Single-container mode (embedded worker) — no redis needed at this scale.
+      GLITCHTIP_EMBED_WORKER: "true"
+      VALKEY_URL: ""
+      ENABLE_USER_REGISTRATION: "false"
+      ENABLE_ORGANIZATION_CREATION: "false"
+      DEFAULT_FROM_EMAIL: ${GLITCHTIP_FROM_EMAIL:-noreply@bennu-api-bahrestan.ir}
+      EMAIL_URL: ${GLITCHTIP_EMAIL_URL:-consolemail://}
+    ports:
+      - "127.0.0.1:8001:8000"
+    depends_on:
+      glitchtip-postgres:
+        condition: service_healthy
+    networks:
+      - monitoring
+      - shared
+      - traefik # public DSN route (profile "public")
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.glitchtip.rule=Host(`${GLITCHTIP_HOST:-errors.localhost}`)
+      - traefik.http.routers.glitchtip.entrypoints=web
+      - traefik.http.services.glitchtip.loadbalancer.server.port=8000
+    restart: unless-stopped
+
+  glitchtip-postgres:
+    image: postgres:16-alpine
+    container_name: monitoring-glitchtip-postgres
+    mem_limit: 128m
+    environment:
+      POSTGRES_DB: glitchtip
+      POSTGRES_USER: glitchtip
+      POSTGRES_PASSWORD: ${GLITCHTIP_DB_PASSWORD:-glitchtip}
+    volumes:
+      - glitchtip_postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U glitchtip -d glitchtip"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  # --------------------------------------------------------------- metrics --
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: monitoring-prometheus
+    mem_limit: 512m
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.retention.time=14d
+      - --storage.tsdb.retention.size=2GB
+    volumes:
+      - ./prometheus:/etc/prometheus:ro
+      - prometheus_data:/prometheus
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: monitoring-grafana
+    mem_limit: 256m
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001/}
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    ports:
+      - "127.0.0.1:3001:3000"
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  # ------------------------------------------------------------------ logs --
+  loki:
+    image: grafana/loki:latest
+    container_name: monitoring-loki
+    mem_limit: 384m
+    command: -config.file=/etc/loki/loki-config.yml
+    volumes:
+      - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro
+      - loki_data:/loki
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: monitoring-promtail
+    mem_limit: 64m
+    command: -config.file=/etc/promtail/promtail-config.yml
+    volumes:
+      - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  # -------------------------------------------------------------- exporters --
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: monitoring-node-exporter
+    mem_limit: 32m
+    command:
+      - --path.rootfs=/host
+    volumes:
+      - /:/host:ro,rslave
+    pid: host
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-postgres-exporter:latest
+    container_name: monitoring-postgres-exporter
+    mem_limit: 48m
+    environment:
+      # App database on the main stack's postgres (reaches it via erp_shared).
+      DATA_SOURCE_NAME: postgres://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/procurement?sslmode=disable
+    networks:
+      - monitoring
+      - shared
+    restart: unless-stopped
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: monitoring-cadvisor
+    mem_limit: 128m
+    privileged: true
+    devices:
+      - /dev/kmsg
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+networks:
+  monitoring:
+    driver: bridge
+  # Reaches the ERP containers (api:8000/metrics, postgres:5432) — the main
+  # compose declares the same fixed-name network as "shared".
+  shared:
+    name: erp_shared
+    external: true
+  # Created by the main compose; required only for the public GlitchTip route.
+  traefik:
+    name: traefik_network
+    external: true
+
+volumes:
+  glitchtip_postgres_data:
+  prometheus_data:
+  grafana_data:
+  loki_data: