- traefik pin v3.3 -> v3.6: v3.3's old docker client fails against
Engine 29 socket proxy (empty daemon errors, no routers)
- backend/minio/chatbot/umami: traefik.docker.network hint — Traefik
picked the internal-network IP for multi-network services and replies
were hairpin-dropped (all backend routes hung)
- db-init: psql needs an explicit -d arameshplaza (no default db 'aramesh'),
set -e so failures actually fail; one-shot folded-YAML needs semicolons
- chatbot image: drop g++ (apt unreachable in build network; all wheels
prebuilt) and mkdir /data before chown (VOLUME does not create it)
- frontend image: create caddy user (alpine image has none) with writable
/config and /data
- monitoring: blackbox probes replace raw metric scrapes of JSON/HTML
health endpoints (they reported 'down' while healthy); blackbox joins
edge; SITE_URL passed via env (Prometheus has no ${VAR:-default}
expansion); ContainerMemoryNearLimit PromQL fixed (comparison inside
label braces); HostDiskAlmostFull filters WSL/Docker-Desktop
pseudo-mounts
- alertmanager: config rendered by stack.sh (no env expansion in AM) and
delivered to Bale (tapi.bale.ai); stack.sh monitoring command
- machine-local override: minio console/API moved to 29001/29000
131 lines
5.5 KiB
YAML
131 lines
5.5 KiB
YAML
# قواعد هشدار — از Prometheus به Alertmanager (تلگرام) میروند.
|
||
# آستانهها محافظهکارانهاند؛ پس از یک هفته داده، دقیقتر تنظیم کنید.
|
||
|
||
groups:
|
||
- name: aramesh-availability
|
||
rules:
|
||
- alert: ServiceDown
|
||
# سرویسهای دارای متریک (backend/traefik/exporters) — بقیه با ProbeFailing پوشش میشوند
|
||
expr: up == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "{{ $labels.job }} پاسخ نمیدهد"
|
||
description: "سرویس {{ $labels.job }} ({{ $labels.instance }}) حداقل ۲ دقیقه پایین است."
|
||
|
||
- alert: ProbeFailing
|
||
expr: probe_success == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "پروب {{ $labels.job }} ناموفق است"
|
||
description: "سرویسِ پشت پروب {{ $labels.instance }} حداقل ۲ دقیقه پاسخ نمیدهد."
|
||
|
||
- alert: BackendHealthFailing
|
||
expr: probe_success{job="blackbox-internal-health"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "بررسی عمق سلامت بکاند شکست میخورد"
|
||
description: "/api/health اتصال DB یا MinIO را آزموده و 503 برگردانده — وابستگیها را ببینید."
|
||
|
||
- alert: PublicSiteDown
|
||
expr: probe_success{job="blackbox-public"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "سایت از مسیر عمومی در دسترس نیست"
|
||
description: "پروب {{ $labels.instance }} ناموفق است — Traefik/TLS/DNS را بررسی کنید."
|
||
|
||
- name: aramesh-resources
|
||
rules:
|
||
- alert: HostDiskAlmostFull
|
||
# Docker Desktop/WSL سوارهای تکراری میسازد — فقط مسیرهای واقعی دیده میشوند
|
||
expr: |
|
||
1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs", mountpoint!~"/run/desktop.*|/parent-distro.*|/usr/lib/wsl.*|/mnt/wsl.*"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs", mountpoint!~"/run/desktop.*|/parent-distro.*|/usr/lib/wsl.*|/mnt/wsl.*"}) > 0.85
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "دیسک میزبان پر میشود"
|
||
description: "پرشدن بیش از ۸۵٪ روی {{ $labels.mountpoint }} — پشتیبانها و لاگها را بررسی کنید."
|
||
|
||
- alert: HostMemoryPressure
|
||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "RAM میزبان رو به تمامشدن است"
|
||
description: "کمتر از ۱۰٪ حافظه باقی مانده است."
|
||
|
||
- alert: ContainerMemoryNearLimit
|
||
expr: |
|
||
(container_memory_usage_bytes{image!="", container!=""}
|
||
/ container_spec_memory_limit_bytes{image!="", container!=""} > 0.9)
|
||
and (container_spec_memory_limit_bytes{image!="", container!=""} > 0)
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "کانتینر {{ $labels.container }} به سقف حافظه نزدیک است"
|
||
|
||
- name: aramesh-app
|
||
rules:
|
||
- alert: Http5xxRateHigh
|
||
expr: |
|
||
sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) by (instance)
|
||
/ sum(rate(http_request_duration_seconds_count[5m])) by (instance) > 0.05
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "نرخ خطای 5xx بکاند بالاست"
|
||
|
||
- alert: Proxy5xxRateHigh
|
||
expr: |
|
||
sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service)
|
||
/ sum(rate(traefik_service_requests_total[5m])) by (service) > 0.05
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "نرخ خطای 5xx روی سرویس {{ $labels.service }} پشت Traefik بالاست"
|
||
|
||
- alert: HighBackendLatency
|
||
expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 2
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "p95 تأخیر بکاند بالای ۲ ثانیه است"
|
||
|
||
- alert: PostgresTooManyConnections
|
||
expr: sum(pg_stat_activity_count) > 80
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "تعداد اتصالهای Postgres نزدیک سقف است"
|
||
|
||
- alert: TlsCertExpiringSoon
|
||
expr: traefik_tls_certs_not_after - time() < 1209600
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "گواهی TLS کمتر از ۱۴ روز اعتبار دارد"
|
||
description: "تمدید خودکار Let's Encrypt کار نکرده — ACME/TLS را بررسی کنید."
|
||
|
||
- alert: BackupStale
|
||
expr: time() - backup_success_time > 93600
|
||
for: 30m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "آخرین پشتیبان موفق بیش از ۲۶ ساعت پیش است"
|
||
description: "کرون/اسکریپت پشتیبانگیری یا سرویس db-backup خفته — فوراً بررسی کنید."
|