fix(stack): bring-up fixes from first local deployment
- traefik pin v3.3 -> v3.6: v3.3's old docker client fails against
Engine 29 socket proxy (empty daemon errors, no routers)
- backend/minio/chatbot/umami: traefik.docker.network hint — Traefik
picked the internal-network IP for multi-network services and replies
were hairpin-dropped (all backend routes hung)
- db-init: psql needs an explicit -d arameshplaza (no default db 'aramesh'),
set -e so failures actually fail; one-shot folded-YAML needs semicolons
- chatbot image: drop g++ (apt unreachable in build network; all wheels
prebuilt) and mkdir /data before chown (VOLUME does not create it)
- frontend image: create caddy user (alpine image has none) with writable
/config and /data
- monitoring: blackbox probes replace raw metric scrapes of JSON/HTML
health endpoints (they reported 'down' while healthy); blackbox joins
edge; SITE_URL passed via env (Prometheus has no ${VAR:-default}
expansion); ContainerMemoryNearLimit PromQL fixed (comparison inside
label braces); HostDiskAlmostFull filters WSL/Docker-Desktop
pseudo-mounts
- alertmanager: config rendered by stack.sh (no env expansion in AM) and
delivered to Bale (tapi.bale.ai); stack.sh monitoring command
- machine-local override: minio console/API moved to 29001/29000
This commit is contained in:
+3
-5
@@ -3,11 +3,8 @@ FROM python:3.11-slim
|
|||||||
|
|
||||||
WORKDIR /srv
|
WORKDIR /srv
|
||||||
|
|
||||||
# وابستگیهای سیستمی برای numpy/scikit-learn
|
# همهٔ وابستگیهای سنگین (numpy/scikit-learn/torch/psycopg) ویل prebuilt دارند —
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
# بدون کامپایلر؛ دسترسی apt در شبکهٔ بیلد قابلاعتماد نیست
|
||||||
g++ \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
COPY requirements.txt /srv/chatbot/requirements.txt
|
COPY requirements.txt /srv/chatbot/requirements.txt
|
||||||
RUN pip install --no-cache-dir -r /srv/chatbot/requirements.txt
|
RUN pip install --no-cache-dir -r /srv/chatbot/requirements.txt
|
||||||
|
|
||||||
@@ -20,6 +17,7 @@ VOLUME /data
|
|||||||
|
|
||||||
# کاربر بدون دسترسی root — فقط /data باید نوشتنی باشد (کش مدلها)
|
# کاربر بدون دسترسی root — فقط /data باید نوشتنی باشد (کش مدلها)
|
||||||
RUN useradd --system --create-home appuser \
|
RUN useradd --system --create-home appuser \
|
||||||
|
&& mkdir -p /data \
|
||||||
&& chown -R appuser:appuser /srv/chatbot /data
|
&& chown -R appuser:appuser /srv/chatbot /data
|
||||||
USER appuser
|
USER appuser
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,9 @@ services:
|
|||||||
- ./monitoring/prometheus:/etc/prometheus:ro
|
- ./monitoring/prometheus:/etc/prometheus:ro
|
||||||
- prometheus_data:/prometheus
|
- prometheus_data:/prometheus
|
||||||
networks: [monitoring, edge]
|
networks: [monitoring, edge]
|
||||||
|
environment:
|
||||||
|
# پروب عمومی — در تولید SITE_URL در .env تنظیم میشود (مثل https://aramesh.example)
|
||||||
|
SITE_URL: ${SITE_URL:-http://frontend/}
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
|
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
@@ -108,7 +111,8 @@ services:
|
|||||||
image: prom/blackbox-exporter:v0.25.0
|
image: prom/blackbox-exporter:v0.25.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
networks: [monitoring]
|
# edge برای رسیدن به سرویسهایی که پروب میشوند (frontend/minio/umami/chatbot/backend)
|
||||||
|
networks: [monitoring, edge]
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
|
|||||||
+15
-3
@@ -19,7 +19,9 @@ services:
|
|||||||
# نقطه ورود واحد — مسیریابی / (سایت استاتیک)، /api (بکاند)، /media (MinIO)،
|
# نقطه ورود واحد — مسیریابی / (سایت استاتیک)، /api (بکاند)، /media (MinIO)،
|
||||||
# /chat (چتبات)، / (آنالیتیکس روی analytics.localhost — پایین همین فایل)
|
# /chat (چتبات)، / (آنالیتیکس روی analytics.localhost — پایین همین فایل)
|
||||||
traefik:
|
traefik:
|
||||||
image: traefik:v3.3
|
# v3.3+ قدیمیترین کلاینت داکر را دارد که با Engineهای جدید (API 1.5x) مشکل دارد —
|
||||||
|
# با v3.6 پین شده است
|
||||||
|
image: traefik:v3.6
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
command:
|
command:
|
||||||
@@ -96,6 +98,8 @@ services:
|
|||||||
- "19000:9000"
|
- "19000:9000"
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
|
# Traefik باید IP لبه را بگیرد — سرویس روی دو شبکه است
|
||||||
|
- traefik.docker.network=aramesh-backend_edge
|
||||||
- traefik.http.routers.media.rule=PathPrefix(`/media`)
|
- traefik.http.routers.media.rule=PathPrefix(`/media`)
|
||||||
- traefik.http.routers.media.entrypoints=web
|
- traefik.http.routers.media.entrypoints=web
|
||||||
- traefik.http.routers.media.priority=100
|
- traefik.http.routers.media.priority=100
|
||||||
@@ -154,9 +158,11 @@ services:
|
|||||||
logging: *default-logging
|
logging: *default-logging
|
||||||
entrypoint: >
|
entrypoint: >
|
||||||
/bin/sh -c "
|
/bin/sh -c "
|
||||||
|
set -e;
|
||||||
for DBN in umami aramesh_chatbot; do
|
for DBN in umami aramesh_chatbot; do
|
||||||
psql -tAc \"SELECT 1 FROM pg_database WHERE datname='$$DBN'\" | grep -q 1 ||
|
if ! psql -d arameshplaza -tAc \"SELECT 1 FROM pg_database WHERE datname='$$DBN'\" | grep -q 1; then
|
||||||
psql -c \"CREATE DATABASE $$DBN\";
|
psql -d arameshplaza -c \"CREATE DATABASE $$DBN\";
|
||||||
|
fi;
|
||||||
echo \"db-init: $$DBN ready\";
|
echo \"db-init: $$DBN ready\";
|
||||||
done
|
done
|
||||||
"
|
"
|
||||||
@@ -198,6 +204,8 @@ services:
|
|||||||
start_period: 20s
|
start_period: 20s
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
|
# Traefik باید IP لبه را بگیرد — سرویس روی دو شبکه است
|
||||||
|
- traefik.docker.network=aramesh-backend_edge
|
||||||
- traefik.http.routers.api.rule=PathPrefix(`/api`)
|
- traefik.http.routers.api.rule=PathPrefix(`/api`)
|
||||||
- traefik.http.routers.api.entrypoints=web
|
- traefik.http.routers.api.entrypoints=web
|
||||||
- traefik.http.routers.api.priority=100
|
- traefik.http.routers.api.priority=100
|
||||||
@@ -268,6 +276,8 @@ services:
|
|||||||
condition: service_completed_successfully
|
condition: service_completed_successfully
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
|
# Traefik باید IP لبه را بگیرد — سرویس روی دو شبکه است
|
||||||
|
- traefik.docker.network=aramesh-backend_edge
|
||||||
- traefik.http.routers.chat.rule=PathPrefix(`/chat`)
|
- traefik.http.routers.chat.rule=PathPrefix(`/chat`)
|
||||||
- traefik.http.routers.chat.entrypoints=web
|
- traefik.http.routers.chat.entrypoints=web
|
||||||
- traefik.http.routers.chat.priority=100
|
- traefik.http.routers.chat.priority=100
|
||||||
@@ -303,6 +313,8 @@ services:
|
|||||||
start_period: 60s
|
start_period: 60s
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
|
# Traefik باید IP لبه را بگیرد — سرویس روی دو شبکه است
|
||||||
|
- traefik.docker.network=aramesh-backend_edge
|
||||||
- traefik.http.routers.analytics.rule=Host(`analytics.localhost`)
|
- traefik.http.routers.analytics.rule=Host(`analytics.localhost`)
|
||||||
- traefik.http.routers.analytics.entrypoints=web
|
- traefik.http.routers.analytics.entrypoints=web
|
||||||
- traefik.http.services.analytics.loadbalancer.server.port=3000
|
- traefik.http.services.analytics.loadbalancer.server.port=3000
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
|
|
||||||
FROM caddy:2-alpine
|
FROM caddy:2-alpine
|
||||||
|
|
||||||
|
# نسخهٔ alpine کاربر caddy ندارد — اینجا ساخته میشود؛ /config و /data مسیرهای
|
||||||
|
# storage سیدی هستند و باید برای کاربر نوشتنی باشند
|
||||||
|
RUN addgroup -S caddy \
|
||||||
|
&& adduser -S -G caddy -s /sbin/nologin caddy \
|
||||||
|
&& mkdir -p /config /data \
|
||||||
|
&& chown -R caddy:caddy /config /data
|
||||||
|
|
||||||
COPY Caddyfile /etc/caddy/Caddyfile
|
COPY Caddyfile /etc/caddy/Caddyfile
|
||||||
COPY --chown=caddy:caddy dist /srv
|
COPY --chown=caddy:caddy dist /srv
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ groups:
|
|||||||
- name: aramesh-availability
|
- name: aramesh-availability
|
||||||
rules:
|
rules:
|
||||||
- alert: ServiceDown
|
- alert: ServiceDown
|
||||||
|
# سرویسهای دارای متریک (backend/traefik/exporters) — بقیه با ProbeFailing پوشش میشوند
|
||||||
expr: up == 0
|
expr: up == 0
|
||||||
for: 2m
|
for: 2m
|
||||||
labels:
|
labels:
|
||||||
@@ -13,6 +14,15 @@ groups:
|
|||||||
summary: "{{ $labels.job }} پاسخ نمیدهد"
|
summary: "{{ $labels.job }} پاسخ نمیدهد"
|
||||||
description: "سرویس {{ $labels.job }} ({{ $labels.instance }}) حداقل ۲ دقیقه پایین است."
|
description: "سرویس {{ $labels.job }} ({{ $labels.instance }}) حداقل ۲ دقیقه پایین است."
|
||||||
|
|
||||||
|
- alert: ProbeFailing
|
||||||
|
expr: probe_success == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "پروب {{ $labels.job }} ناموفق است"
|
||||||
|
description: "سرویسِ پشت پروب {{ $labels.instance }} حداقل ۲ دقیقه پاسخ نمیدهد."
|
||||||
|
|
||||||
- alert: BackendHealthFailing
|
- alert: BackendHealthFailing
|
||||||
expr: probe_success{job="blackbox-internal-health"} == 0
|
expr: probe_success{job="blackbox-internal-health"} == 0
|
||||||
for: 2m
|
for: 2m
|
||||||
@@ -34,8 +44,9 @@ groups:
|
|||||||
- name: aramesh-resources
|
- name: aramesh-resources
|
||||||
rules:
|
rules:
|
||||||
- alert: HostDiskAlmostFull
|
- alert: HostDiskAlmostFull
|
||||||
|
# Docker Desktop/WSL سوارهای تکراری میسازد — فقط مسیرهای واقعی دیده میشوند
|
||||||
expr: |
|
expr: |
|
||||||
1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"}) > 0.85
|
1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs", mountpoint!~"/run/desktop.*|/parent-distro.*|/usr/lib/wsl.*|/mnt/wsl.*"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs", mountpoint!~"/run/desktop.*|/parent-distro.*|/usr/lib/wsl.*|/mnt/wsl.*"}) > 0.85
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
@@ -54,8 +65,9 @@ groups:
|
|||||||
|
|
||||||
- alert: ContainerMemoryNearLimit
|
- alert: ContainerMemoryNearLimit
|
||||||
expr: |
|
expr: |
|
||||||
container_memory_usage_bytes{image!="", container!="", container_spec_memory_limit_bytes > 0}
|
(container_memory_usage_bytes{image!="", container!=""}
|
||||||
/ container_spec_memory_limit_bytes{container!="", container_spec_memory_limit_bytes > 0} > 0.9
|
/ container_spec_memory_limit_bytes{image!="", container!=""} > 0.9)
|
||||||
|
and (container_spec_memory_limit_bytes{image!="", container!=""} > 0)
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|||||||
@@ -39,26 +39,49 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["postgres-exporter:9187"]
|
- targets: ["postgres-exporter:9187"]
|
||||||
|
|
||||||
# سلامت مستقیم سرویسها — up == 0 یعنی آن سرویس پایین است
|
# سلامت سرویسهای بدون متریک — با پروب blackbox (پاسخ JSON/HTML قابل اسکرپ نیست)
|
||||||
- job_name: frontend
|
# نتیجه: probe_success (۱ = سالم، ۰ = پایین) — هشدار ProbeFailing پوشش میدهد
|
||||||
metrics_path: /
|
- job_name: blackbox-frontend
|
||||||
|
metrics_path: /probe
|
||||||
|
params:
|
||||||
|
module: [http_2xx]
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["frontend:80"]
|
- targets:
|
||||||
|
- http://frontend/
|
||||||
|
relabel_configs: &blackbox_relabel
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: blackbox-exporter:9115
|
||||||
|
|
||||||
- job_name: minio
|
- job_name: blackbox-minio
|
||||||
metrics_path: /minio/health/live
|
metrics_path: /probe
|
||||||
|
params:
|
||||||
|
module: [http_2xx]
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["minio:9000"]
|
- targets:
|
||||||
|
- http://minio:9000/minio/health/live
|
||||||
|
relabel_configs: *blackbox_relabel
|
||||||
|
|
||||||
- job_name: umami
|
- job_name: blackbox-umami
|
||||||
metrics_path: /api/heartbeat
|
metrics_path: /probe
|
||||||
|
params:
|
||||||
|
module: [http_2xx]
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["umami:3000"]
|
- targets:
|
||||||
|
- http://umami:3000/api/heartbeat
|
||||||
|
relabel_configs: *blackbox_relabel
|
||||||
|
|
||||||
- job_name: chatbot
|
- job_name: blackbox-chatbot
|
||||||
metrics_path: /chat/healthz
|
metrics_path: /probe
|
||||||
|
params:
|
||||||
|
module: [http_2xx]
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["chatbot:8000"]
|
- targets:
|
||||||
|
- http://chatbot:8000/chat/healthz
|
||||||
|
relabel_configs: *blackbox_relabel
|
||||||
|
|
||||||
# پروب بیروننگر — سلامت واقعی /api/health (بخش عمق: DB و MinIO را میآزماید)
|
# پروب بیروننگر — سلامت واقعی /api/health (بخش عمق: DB و MinIO را میآزماید)
|
||||||
# و در تولید، آدرس عمومی سایت (SITE_URL از .env — پشتیبانی متغیر محیطی Prometheus)
|
# و در تولید، آدرس عمومی سایت (SITE_URL از .env — پشتیبانی متغیر محیطی Prometheus)
|
||||||
@@ -69,13 +92,7 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets:
|
- targets:
|
||||||
- http://backend:8000/api/health
|
- http://backend:8000/api/health
|
||||||
relabel_configs: &blackbox_relabel
|
relabel_configs: *blackbox_relabel
|
||||||
- source_labels: [__address__]
|
|
||||||
target_label: __param_target
|
|
||||||
- source_labels: [__param_target]
|
|
||||||
target_label: instance
|
|
||||||
- target_label: __address__
|
|
||||||
replacement: blackbox-exporter:9115
|
|
||||||
|
|
||||||
- job_name: blackbox-public
|
- job_name: blackbox-public
|
||||||
metrics_path: /probe
|
metrics_path: /probe
|
||||||
@@ -83,5 +100,7 @@ scrape_configs:
|
|||||||
module: [http_2xx]
|
module: [http_2xx]
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets:
|
- targets:
|
||||||
- ${SITE_URL:-http://frontend/}
|
# مقدار پیشفرض در docker-compose.monitoring.yml تنظیم میشود
|
||||||
|
# (Prometheus سینتکس ${VAR:-default} را پشتیبانی نمیکند)
|
||||||
|
- ${SITE_URL}
|
||||||
relabel_configs: *blackbox_relabel
|
relabel_configs: *blackbox_relabel
|
||||||
|
|||||||
Reference in New Issue
Block a user