From 887a3af84fd97582a73f589c0196e3a0cf4f09f9 Mon Sep 17 00:00:00 2001 From: "reza.g" Date: Tue, 27 Oct 2026 01:43:31 +0330 Subject: [PATCH] refactor(monitoring): move grafana into the main compose stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner request: one 'docker compose up -d' should bring the whole system up — no separate monitoring project to start for the dashboards. - docker-compose.yml: grafana service (procurement-grafana, loopback :3001, default+shared networks, obs datasource env) + grafana_data volume; provisioning and dashboard JSONs stay in ./monitoring/grafana/ (no file moves, provisioning unchanged) - docker-compose.prod.yml: GRAFANA_ADMIN_PASSWORD forced via the existing ${VAR:?} pattern (local keeps a lenient default, prod refuses defaults) - monitoring/docker-compose.monitoring.yml: grafana service + volume removed (no name/port conflict); glitchtip/prometheus/loki stay server-side as before - docs updated: grafana restarts on app deploys are harmless (settings live in the volume); SSH-tunnel posture unchanged Verified locally: both compose files validate; 'docker compose up -d' starts all 10 services incl. procurement-grafana; BotPostgres datasource health OK; all 3 dashboards provisioned in the fresh volume; monitoring compose no longer contains grafana; old monitoring-grafana container removed; port still loopback-only. --- bot/docs/DEPLOY-AND-MONITORING.md | 9 ++++--- docker-compose.prod.yml | 4 +++ docker-compose.yml | 32 ++++++++++++++++++++++++ monitoring/README.md | 17 +++++++++---- monitoring/docker-compose.monitoring.yml | 32 ++++-------------------- 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/bot/docs/DEPLOY-AND-MONITORING.md b/bot/docs/DEPLOY-AND-MONITORING.md index d52b938..6335194 100644 --- a/bot/docs/DEPLOY-AND-MONITORING.md +++ b/bot/docs/DEPLOY-AND-MONITORING.md @@ -60,10 +60,11 @@ → قیف باید پر شود و رخدادهای همان پیام‌ها در `bot_trace_events` باشد. 5. چک سهمیه: `usage` در `jev.result` (توکن ورودی/خروجی) + latency p95. 6. **داشبوردها (به‌روز ۲۰۲۶-۱۰-۲۵، v2.1):** `PROCUREMENT_BOT_OBS_READER_PASSWORD` را در `.env` ست کنید و - یک بار بات را restart کنید تا نقش فقط‌خواندنی `obs_reader` ساخته شود؛ سپس گرفانا را - (ری)استارت کنید — دیتاسورس `BotPostgres` و دو داشبورد «بات — Observability خط لوله» و - «بات — اکسپلورر پیام‌ها» خودکار provision می‌شوند (جزئیات در `monitoring/README.md`). - صفحه‌ی فارسی داخل ERP روی `/bot-monitor` (فقط ادمین) چهار تب دارد: نمای کلی، + یک بار بات را restart کنید تا نقش فقط‌خواندنی `obs_reader` ساخته شود. **گرفانا حالا جزو استک اصلی است** + (تصمیم ۲۰۲۶-۱۰-۲۵: یک `docker compose up -d` همه‌چیز از جمله گرفانا را بالا می‌آورد؛ در پروداکشن + overlay، رمز `GRAFANA_ADMIN_PASSWORD` الزامی است). دیتاسورس `BotPostgres` و داشبوردهای + «بات — Observability خط لوله» و «بات — اکسپلورر پیام‌ها» خودکار provision می‌شوند + (جزئیات در `monitoring/README.md`). صفحه‌ی فارسی داخل ERP روی `/bot-monitor` (فقط ادمین) چهار تب دارد: نمای کلی، **مدل‌ها** (رکورد و روند زمانی هر مدل: Jev، DeepSeek-NLU، داور، فالبک، هر STT، بهبود صدا)، **بخش‌ها** (مراحل به تفکیک بخش خط‌لوله) و **پیام‌ها** (پرونده‌ی هر پیام با پخش/دانلود ویس و ترنسکریپت همه‌ی موتورها کنار هم). diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 0a2d83a..775abbf 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,6 +26,10 @@ services: environment: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + grafana: + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} + minio: environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:?set MINIO_ROOT_USER in .env} diff --git a/docker-compose.yml b/docker-compose.yml index 822ef6a..b0ab7dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,6 +362,37 @@ services: restart: unless-stopped mem_limit: 512m + # ---------------------------------------------------------------- grafana -- + # Bot-observability dashboards — part of the MAIN stack (owner decision + # 2026-10-25: one `docker compose up -d` brings everything up). Provisioning + # and dashboard JSONs live in ./monitoring/grafana/ and stay shared with the + # server-side monitoring docs. Loopback-only, reads Postgres via obs_reader. + grafana: + image: grafana/grafana:11.6.3 + container_name: procurement-grafana + mem_limit: 256m + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-grafana-admin} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001/} + # Bot-observability Postgres datasource (provisioned from datasources.yml) + OBS_DB_HOST: postgres + OBS_DB_PORT: "5432" + OBS_DB_NAME: procurement + OBS_DB_USER: ${OBS_READER_USER:-obs_reader} + OBS_DB_PASSWORD: ${PROCUREMENT_BOT_OBS_READER_PASSWORD:-} + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "127.0.0.1:3001:3000" + networks: + - default + - shared # reaches postgres for the bot_traces/bot_trace_events datasource + restart: unless-stopped + networks: # شبکه‌ی عمومی: فقط ترافیک و سرویس‌های عمومی. postgres روی شبکه‌ی # پیش‌فرض می‌ماند و از بیرون قابل دسترسی نیست. @@ -378,3 +409,4 @@ volumes: postgres_data: minio_data: traefik_certs: + grafana_data: diff --git a/monitoring/README.md b/monitoring/README.md index 4982f85..f9d70be 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -21,10 +21,12 @@ cd /srv/erp # the repo checkout used by the deploy webhook # GLITCHTIP_SECRET_KEY, GLITCHTIP_DOMAIN/HOST, GLITCHTIP_DB_PASSWORD, # GRAFANA_ADMIN_PASSWORD, POSTGRES_PASSWORD (already set), LOG_FORMAT=json -# 2) the main compose gained a shared network — re-up once so it exists: +# 2) the main compose gained a shared network — re-up once so it exists. +# NOTE (2026-10-25): GRAFANA MOVED into the main docker-compose.yml — +# this one command now brings the app AND grafana up together: docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d -# 3) start monitoring: +# 3) start the REST of monitoring (glitchtip/prometheus/loki — no grafana): docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d # 4) GlitchTip is routed publicly through Traefik for the browser SDK: @@ -36,7 +38,9 @@ Admin UIs stay **loopback-only** (same posture as the MinIO console): ```powershell ssh -L 3001:127.0.0.1:3001 -L 8001:127.0.0.1:8001 @ -# Grafana → http://localhost:3001 (dashboards auto-provisioned) +# Grafana → http://localhost:3001 (runs in the MAIN app stack since +# 2026-10-25; dashboards auto-provisioned; restarts on app deploys +# are harmless — settings live in the grafana_data volume) # GlitchTip → http://localhost:8001 (create the first admin on first open) ``` @@ -156,8 +160,11 @@ webhook uses it post-rebuild regardless of the metrics stack. ## Local runs (Docker Desktop) -Start WITHOUT `--profile host-metrics` — node-exporter and cAdvisor need -Linux rootfs mounts and will fail to start on Windows/macOS hosts: +Grafana starts with the app stack — plain `docker compose up -d` from the +repo root brings api/frontend/bot/postgres/… **and grafana** (loopback +:3001). For the rest of the monitoring stack start WITHOUT +`--profile host-metrics` — node-exporter and cAdvisor need Linux rootfs +mounts and will fail to start on Windows/macOS hosts: ```bash docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d diff --git a/monitoring/docker-compose.monitoring.yml b/monitoring/docker-compose.monitoring.yml index 38dc4df..7626eeb 100644 --- a/monitoring/docker-compose.monitoring.yml +++ b/monitoring/docker-compose.monitoring.yml @@ -85,33 +85,12 @@ services: - shared restart: unless-stopped - grafana: - image: grafana/grafana:11.6.3 - container_name: monitoring-grafana - mem_limit: 256m - environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} - GF_USERS_ALLOW_SIGN_UP: "false" - GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001/} - # Bot-observability Postgres datasource (provisioned from datasources.yml) - OBS_DB_HOST: postgres - OBS_DB_PORT: "5432" - OBS_DB_NAME: procurement - OBS_DB_USER: ${OBS_READER_USER:-obs_reader} - OBS_DB_PASSWORD: ${PROCUREMENT_BOT_OBS_READER_PASSWORD:-} - volumes: - - grafana_data:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning:ro - - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - ports: - - "127.0.0.1:3001:3000" - networks: - - monitoring - - shared # reaches the app postgres for the bot_traces/bot_trace_events datasource - restart: unless-stopped - # ------------------------------------------------------------------ logs -- + # NOTE (2026-10-25): the grafana service MOVED to the main docker-compose.yml + # (owner decision: one `docker compose up -d` brings the app + grafana up + # together). It still reads its provisioning + dashboards from + # ./monitoring/grafana/ — see the main compose file for env/ports. + loki: image: grafana/loki:3.5.0 container_name: monitoring-loki @@ -202,5 +181,4 @@ networks: volumes: glitchtip_postgres_data: prometheus_data: - grafana_data: loki_data: