feat(stack): full overhaul — hardening, ops tooling, monitoring, analytics, chatbot

Security: deep /api/health (DB+MinIO probes, 503 on failure) + /api/live; rate-limited public forms with honeypots; real client IP behind Traefik (--proxy-headers); TipTap URL sanitization on write (nh3); non-root containers + slim .dockerignore; prod MinIO console bound to 127.0.0.1; HSTS/nosniff on TLS routers; JWT 8h; no default credentials in code. Reliability: Alembic migrations (create_all baseline); graceful shutdown (exec uvicorn); hardened backups (checksums, sanity check, freshness gauge, optional rclone off-site) + restore.sh + hourly db-backup sidecar (ops profile); stack.sh ops tooling (deploy/logs/health/backup/restore/monitoring); .gitattributes LF for shell scripts. Orchestration: json-file log rotation, pinned images, per-service memory limits, edge/internal networks, Traefik ping/accesslog/metrics + flood limiter, conditional minio-setup, pgvector db image, one-shot db-init (umami + aramesh_chatbot databases). Chatbot: unified from site-chatbot branch (RAG + pgvector, SSE, ChatWidget). Monitoring: Prometheus/Alertmanager/Loki/Promtail/Grafana + cAdvisor, node/postgres/blackbox exporters, provisioned dashboards and 10 alert rules; alerts delivered to Bale (tapi.bale.ai — Telegram-compatible). Analytics: Umami on shared Postgres, build-time tracker injection, custom events (contact/newsletter/phone/chat/lang) via frontend/src/analytics.ts. Hygiene: CSV export for contact messages and subscribers; upload image optimization (Pillow); CI script + GitHub-compatible workflow; audit.sh; gen-env.sh for strong credentials.
2026-09-19 00:24:57 +03:30
parent 805f4594d4
commit ce0735d775
75 changed files with 4287 additions and 153 deletions
@@ -0,0 +1,24 @@
+# Alertmanager — تحویل هشدارها به بله (API بات بله همان شکل تلگرام است)
+# توکن و شناسه چت از .env (ALERT_BOT_TOKEN / ALERT_CHAT_ID) — این فایل قالب است؛
+# نسخهٔ رندرشده (alertmanager.rendered.yml) را scripts/stack.sh می‌سازد و در گیت نمی‌رود.
+# تا وقتی توکن/چت واقعی تنظیم نشده، ارسال شکست می‌خورد و هشدارها در
+# http://<prometheus>:9090/alerts دیده می‌شوند.
+
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: bale
+  group_by: [alertname, job]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: bale
+    telegram_configs:
+      # تنها تفاوت با تلگرام: نشانی API بات بله
+      - api_url: https://tapi.bale.ai
+        bot_token: "${ALERT_BOT_TOKEN}"
+        chat_id: ${ALERT_CHAT_ID}
+        send_resolved: true
@@ -0,0 +1,102 @@
+{
+  "uid": "aramesh-app",
+  "title": "آرامش — اپلیکیشن و ترافیک",
+  "tags": ["aramesh"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "درخواست‌ها بر ثانیه (هر روتر Traefik)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "sum(rate(traefik_service_requests_total[5m])) by (service)",
+          "legendFormat": "{{ service }}"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "خطاهای 5xx (بک‌اند)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "sum(rate(http_request_duration_seconds_count{status=~\"5..\"}[5m]))",
+          "legendFormat": "5xx/s"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "تأخیر بک‌اند — p50 و p95",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
+          "legendFormat": "p95"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }
+    },
+    {
+      "type": "stat",
+      "title": "سلامت /api/health (پروب عمق)",
+      "gridPos": { "h": 8, "w": 4, "x": 12, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        { "expr": "probe_success{job=\"blackbox-internal-health\"}", "legendFormat": "health" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            { "type": "value", "options": { "1": { "text": "سالم", "color": "green" }, "0": { "text": "خراب", "color": "red" } } }
+          ],
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "تازگی آخرین پشتیبان (ثانیه)",
+      "gridPos": { "h": 8, "w": 4, "x": 16, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        { "expr": "time() - backup_success_time", "legendFormat": "age" }
+      ],
+      "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 72000 }, { "color": "red", "value": 93600 } ] } }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "اتصال‌های Postgres",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        { "expr": "sum(pg_stat_activity_count)", "legendFormat": "اتصال‌ها" }
+      ],
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }
+    },
+    {
+      "type": "logs",
+      "title": "لاگ زندهٔ سرویس‌ها (Loki)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "targets": [
+        { "expr": "{service=~\"backend|traefik|chatbot|umami\"}" }
+      ]
+    }
+  ]
+}
@@ -0,0 +1,74 @@
+{
+  "uid": "aramesh-infra",
+  "title": "آرامش — زیرساخت (میزبان و کانتینرها)",
+  "tags": ["aramesh"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "CPU میزبان",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "CPU %"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "percent", "max": 100 }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "حافظه میزبان",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        { "expr": "node_memory_MemTotal_bytes", "legendFormat": "کل" },
+        { "expr": "node_memory_MemAvailable_bytes", "legendFormat": "آزاد" }
+      ],
+      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "دیسک — درصد پرشدگی",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\"}))",
+          "legendFormat": "{{ mountpoint }}"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "percent", "max": 100 }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "مصرف RAM هر کانتینر",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "topk(10, container_memory_usage_bytes{image!=\"\", container!=\"\"})",
+          "legendFormat": "{{ container }}"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU هر کانتینر",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prom" },
+      "targets": [
+        {
+          "expr": "topk(10, rate(container_cpu_usage_seconds_total{image!=\"\", container!=\"\"}[5m]))",
+          "legendFormat": "{{ container }}"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }
+    }
+  ]
+}
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: aramesh
+    folder: Aramesh
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false
@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prom
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Loki
+    type: loki
+    uid: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
@@ -0,0 +1,30 @@
+# Loki — ذخیرهٔ لاگ‌ها روی دیسک (۱۴ روز)
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 336h
@@ -0,0 +1,118 @@
+# قواعد هشدار — از Prometheus به Alertmanager (تلگرام) می‌روند.
+# آستانه‌ها محافظه‌کارانه‌اند؛ پس از یک هفته داده، دقیق‌تر تنظیم کنید.
+
+groups:
+  - name: aramesh-availability
+    rules:
+      - alert: ServiceDown
+        expr: up == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.job }} پاسخ نمی‌دهد"
+          description: "سرویس {{ $labels.job }} ({{ $labels.instance }}) حداقل ۲ دقیقه پایین است."
+
+      - alert: BackendHealthFailing
+        expr: probe_success{job="blackbox-internal-health"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "بررسی عمق سلامت بک‌اند شکست می‌خورد"
+          description: "‏/api/health اتصال DB یا MinIO را آزموده و 503 برگردانده — وابستگی‌ها را ببینید."
+
+      - alert: PublicSiteDown
+        expr: probe_success{job="blackbox-public"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "سایت از مسیر عمومی در دسترس نیست"
+          description: "پروب {{ $labels.instance }} ناموفق است — Traefik/TLS/DNS را بررسی کنید."
+
+  - name: aramesh-resources
+    rules:
+      - alert: HostDiskAlmostFull
+        expr: |
+          1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"}) > 0.85
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "دیسک میزبان پر می‌شود"
+          description: "پرشدن بیش از ۸۵٪ روی {{ $labels.mountpoint }} — پشتیبان‌ها و لاگ‌ها را بررسی کنید."
+
+      - alert: HostMemoryPressure
+        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "RAM میزبان رو به تمام‌شدن است"
+          description: "کمتر از ۱۰٪ حافظه باقی مانده است."
+
+      - alert: ContainerMemoryNearLimit
+        expr: |
+          container_memory_usage_bytes{image!="", container!="", container_spec_memory_limit_bytes > 0}
+            / container_spec_memory_limit_bytes{container!="", container_spec_memory_limit_bytes > 0} > 0.9
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "کانتینر {{ $labels.container }} به سقف حافظه نزدیک است"
+
+  - name: aramesh-app
+    rules:
+      - alert: Http5xxRateHigh
+        expr: |
+          sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) by (instance)
+            / sum(rate(http_request_duration_seconds_count[5m])) by (instance) > 0.05
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "نرخ خطای 5xx بک‌اند بالاست"
+
+      - alert: Proxy5xxRateHigh
+        expr: |
+          sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service)
+            / sum(rate(traefik_service_requests_total[5m])) by (service) > 0.05
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "نرخ خطای 5xx روی سرویس {{ $labels.service }} پشت Traefik بالاست"
+
+      - alert: HighBackendLatency
+        expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 2
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "p95 تأخیر بک‌اند بالای ۲ ثانیه است"
+
+      - alert: PostgresTooManyConnections
+        expr: sum(pg_stat_activity_count) > 80
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "تعداد اتصال‌های Postgres نزدیک سقف است"
+
+      - alert: TlsCertExpiringSoon
+        expr: traefik_tls_certs_not_after - time() < 1209600
+        labels:
+          severity: warning
+        annotations:
+          summary: "گواهی TLS کمتر از ۱۴ روز اعتبار دارد"
+          description: "تمدید خودکار Let's Encrypt کار نکرده — ACME/TLS را بررسی کنید."
+
+      - alert: BackupStale
+        expr: time() - backup_success_time > 93600
+        for: 30m
+        labels:
+          severity: critical
+        annotations:
+          summary: "آخرین پشتیبان موفق بیش از ۲۶ ساعت پیش است"
+          description: "کرون/اسکریپت پشتیبان‌گیری یا سرویس db-backup خفته — فوراً بررسی کنید."
@@ -0,0 +1,87 @@
+# تنظیمات Prometheus — اسکرپ همهٔ سرویس‌های استک از شبکه‌های داخلی
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  # بک‌اند FastAPI — /metrics با instrumentator (فقط از شبکه داخلی)
+  - job_name: backend
+    static_configs:
+      - targets: ["backend:8000"]
+
+  # پراکسی — RPS هر روتر، کدهای وضعیت، انقضای گواهی TLS
+  - job_name: traefik
+    static_configs:
+      - targets: ["traefik:8080"]
+
+  # منابع کانتینرها و میزبان
+  - job_name: cadvisor
+    static_configs:
+      - targets: ["cadvisor:8080"]
+
+  - job_name: node
+    static_configs:
+      - targets: ["node-exporter:9100"]
+
+  - job_name: postgres
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+
+  # سلامت مستقیم سرویس‌ها — up == 0 یعنی آن سرویس پایین است
+  - job_name: frontend
+    metrics_path: /
+    static_configs:
+      - targets: ["frontend:80"]
+
+  - job_name: minio
+    metrics_path: /minio/health/live
+    static_configs:
+      - targets: ["minio:9000"]
+
+  - job_name: umami
+    metrics_path: /api/heartbeat
+    static_configs:
+      - targets: ["umami:3000"]
+
+  - job_name: chatbot
+    metrics_path: /chat/healthz
+    static_configs:
+      - targets: ["chatbot:8000"]
+
+  # پروب بیرون‌نگر — سلامت واقعی /api/health (بخش عمق: DB و MinIO را می‌آزماید)
+  # و در تولید، آدرس عمومی سایت (SITE_URL از .env — پشتیبانی متغیر محیطی Prometheus)
+  - job_name: blackbox-internal-health
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - http://backend:8000/api/health
+    relabel_configs: &blackbox_relabel
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+
+  - job_name: blackbox-public
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - ${SITE_URL:-http://frontend/}
+    relabel_configs: *blackbox_relabel
@@ -0,0 +1,27 @@
+# Promtail — لاگ همهٔ کانتینرهای استک آرامش را از Docker socket می‌خواند
+server:
+  http_listen_port: 9080
+
+positions:
+  filename: /positions/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        # فقط کانتینرهای همین استک — نه بقیهٔ کانتینرهای میزبان
+        filters:
+          - name: label
+            values: ["com.docker.compose.project=aramesh-backend"]
+    relabel_configs:
+      - source_labels: ["__meta_docker_container_name"]
+        regex: "/(.*)"
+        target_label: container
+      - source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
+        target_label: service
+      - source_labels: ["__meta_docker_container_log_stream"]
+        target_label: stream