From e17d1f0883571f82aac0b7494b49bd953324e6fb Mon Sep 17 00:00:00 2001 From: MMD Date: Fri, 18 Sep 2026 22:18:02 +0330 Subject: [PATCH] feat(monitoring): observability stack + app instrumentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys --- .env.example | 22 +- backend/app/config.py | 2 + backend/app/main.py | 80 +++++++- backend/app/observability.py | 86 ++++++++ backend/app/routers/auth.py | 10 + backend/app/routers/pm.py | 11 + backend/app/services/analytics.py | 76 +++++++ backend/app/services/quote_service.py | 15 ++ docker-compose.yml | 31 +++ frontend/Dockerfile | 9 + frontend/package.json | 2 + frontend/src/components/ErrorBoundary.tsx | 3 + frontend/src/lib/analytics.ts | 80 ++++++++ frontend/src/main.tsx | 4 + frontend/src/pages/Auth/LoginPage.tsx | 4 + frontend/src/pages/Wizard/WizardPage.tsx | 3 + frontend/src/vite-env.d.ts | 9 + monitoring/README.md | 111 ++++++++++ monitoring/docker-compose.monitoring.yml | 191 ++++++++++++++++++ .../grafana/dashboards/erp-overview.json | 159 +++++++++++++++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/datasources.yml | 15 ++ monitoring/loki/loki-config.yml | 43 ++++ monitoring/prometheus/alerts.yml | 58 ++++++ monitoring/prometheus/prometheus.yml | 34 ++++ monitoring/promtail/promtail-config.yml | 40 ++++ 26 files changed, 1106 insertions(+), 3 deletions(-) create mode 100644 backend/app/observability.py create mode 100644 backend/app/services/analytics.py create mode 100644 frontend/src/lib/analytics.ts create mode 100644 monitoring/README.md create mode 100644 monitoring/docker-compose.monitoring.yml create mode 100644 monitoring/grafana/dashboards/erp-overview.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/datasources.yml create mode 100644 monitoring/loki/loki-config.yml create mode 100644 monitoring/prometheus/alerts.yml create mode 100644 monitoring/prometheus/prometheus.yml create mode 100644 monitoring/promtail/promtail-config.yml diff --git a/.env.example b/.env.example index bc4ca55..f325d96 100644 --- a/.env.example +++ b/.env.example @@ -40,9 +40,27 @@ DEV_LOGIN_ENABLED=false # --- Deploy webhook (Gitea push → rebuild; see infra/deploy-webhook/) --- GITEA_WEBHOOK_SECRET=CHANGE_ME_long_random_webhook_secret -# --- Monitoring stack (see monitoring/docker-compose.monitoring.yml) --- +# --- Monitoring stack (see monitoring/README.md) --- +# API log format for Loki ingestion ("text" locally, "json" in production) +LOG_FORMAT=json +# GlitchTip (error tracking). GLITCHTIP_HOST must be a public DNS name — +# the browser SDK posts errors there. +GLITCHTIP_SECRET_KEY=CHANGE_ME_random_django_secret +GLITCHTIP_DOMAIN=https://errors.bennu-api-bahrestan.ir +GLITCHTIP_HOST=errors.bennu-api-bahrestan.ir +GLITCHTIP_DB_PASSWORD=CHANGE_ME_glitchtip_db_password +GLITCHTIP_FROM_EMAIL=noreply@bennu-api-bahrestan.ir +GLITCHTIP_EMAIL_URL=consolemail:// +# Grafana (dashboards) — UI is loopback-only (:3001) unless you add Traefik labels +GRAFANA_ADMIN_PASSWORD=CHANGE_ME_grafana_password +GRAFANA_ROOT_URL=http://localhost:3001/ +# OpenPanel (product analytics) — client id/secret from the OpenPanel project +OPENPANEL_API_URL=http://openpanel-api:3333 OPENPANEL_CLIENT_ID= OPENPANEL_CLIENT_SECRET= +# Public origin of the OpenPanel dashboard/API the BROWSER talks to +# (e.g. https://stats.bennu-api-bahrestan.ir — see monitoring/README.md) +OPENPANEL_PUBLIC_URL= +# Sentry/GlitchTip DSNs created in the GlitchTip UI after first boot SENTRY_DSN_BACKEND= SENTRY_DSN_FRONTEND= -GRAFANA_ADMIN_PASSWORD=CHANGE_ME_grafana_password diff --git a/backend/app/config.py b/backend/app/config.py index 157b7ba..c2d666a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -78,6 +78,8 @@ class Settings(BaseSettings): INVENTORY_DEFAULT_PROJECT_ID: str = "" # ---- Observability (Phase 2) ------------------------------------------- + # "text" (human console) | "json" (JSON lines for Loki/Promtail). + LOG_FORMAT: str = "text" # Sentry/GlitchTip DSN — backend exception capture (empty = disabled). SENTRY_DSN: str = "" SENTRY_TRACES_SAMPLE_RATE: float = 0.0 diff --git a/backend/app/main.py b/backend/app/main.py index 10529da..090acfe 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -7,12 +7,16 @@ """ import asyncio import logging +import uuid from contextlib import asynccontextmanager +import httpx from fastapi import Depends, FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from prometheus_fastapi_instrumentator import Instrumentator +from sqlalchemy import text as sql_text from sqlalchemy.exc import IntegrityError from app.auth import get_current_user @@ -25,13 +29,29 @@ from app.db.init_db import ( ensure_quote_projects, ensure_v26, ) +from app.observability import RequestContextMiddleware, setup_logging from app.routers import ROUTERS from app.services.quote_service import auto_expire_quotes from app.utils.errors import AppError -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s") +setup_logging(settings.LOG_FORMAT) logger = logging.getLogger("procurement") +if settings.SENTRY_DSN: + # GlitchTip/Sentry exception capture — after setup_logging so the SDK's + # own logs go through our formatter too. Disabled entirely without a DSN. + import sentry_sdk + from sentry_sdk.integrations.fastapi import FastApiIntegration + + sentry_sdk.init( + dsn=settings.SENTRY_DSN, + environment=settings.ENVIRONMENT, + traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE, + integrations=[FastApiIntegration()], + send_default_pii=False, + ) + logger.info("Sentry/GlitchTip exception capture enabled") + API_V1_PREFIX = "/api/v1" @@ -107,6 +127,12 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) +app.add_middleware(RequestContextMiddleware) + +# Prometheus /metrics (requests, latency, errors) — mounted at app level, +# outside the authenticated /api/v1 prefix; scrape target for the +# monitoring stack. +Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False) # --------------------------------------------------------------------------- @@ -203,3 +229,55 @@ for router in ROUTERS: @app.get("/health", tags=["system"]) async def health(): return {"status": "ok", "service": "procurement-api"} + + +async def _check_db() -> bool: + try: + async with AsyncSessionLocal() as session: + await session.execute(sql_text("SELECT 1")) + return True + except Exception: + logger.exception("readiness: database check failed") + return False + + +async def _check_minio() -> bool: + try: + from app.services.storage import get_client + + get_client().bucket_exists(settings.MINIO_BUCKET) + return True + except Exception: + logger.exception("readiness: MinIO check failed") + return False + + +async def _check_casdoor() -> bool: + try: + async with httpx.AsyncClient(timeout=5) as client: + resp = await client.get( + settings.CASDOOR_JWKS_URL or f"{settings.CASDOOR_ENDPOINT}/.well-known/jwks" + ) + return resp.status_code == 200 + except Exception: + logger.exception("readiness: Casdoor JWKS check failed") + return False + + +@app.get("/health/ready", tags=["system"]) +async def health_ready(): + """Readiness: verifies the dependencies the API actually needs. + + The deploy webhook polls this after a rebuild, so a deploy that breaks + the DB/MinIO/Casdoor connectivity is visible immediately. + """ + checks = { + "database": await _check_db(), + "minio": await _check_minio(), + "casdoor": await _check_casdoor(), + } + ok = all(checks.values()) + return JSONResponse( + status_code=200 if ok else 503, + content={"status": "ok" if ok else "degraded", "service": "procurement-api", "checks": checks}, + ) diff --git a/backend/app/observability.py b/backend/app/observability.py new file mode 100644 index 0000000..66f439f --- /dev/null +++ b/backend/app/observability.py @@ -0,0 +1,86 @@ +"""Observability plumbing — logging format, request IDs, request log. + +- ``setup_logging()``: human-readable text by default, JSON lines when + ``PROCUREMENT_LOG_FORMAT=json`` (what Promtail/Loki prefers). +- ``request_context`` middleware: assigns an X-Request-ID to every request + (honours one sent by the edge), echoes it in the response, and logs one + structured line per request (method, path, status, duration, request id) + — this is what makes access logs useful in Loki. +""" +import contextvars +import json +import logging +import time +import uuid + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +request_id_var: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="-") + +_SKIP_ACCESS_LOG = {"/health", "/health/ready", "/metrics"} + + +class JsonFormatter(logging.Formatter): + """Minimal JSON-lines formatter for Loki/Promtail ingestion.""" + + def format(self, record: logging.LogRecord) -> str: + payload = { + "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "request_id": request_id_var.get(), + } + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + for key in ("http_method", "http_path", "http_status", "duration_ms"): + value = getattr(record, key, None) + if value is not None: + payload[key] = value + return json.dumps(payload, ensure_ascii=False) + + +def setup_logging(log_format: str = "text", level: int = logging.INFO) -> None: + formatter = JsonFormatter() if log_format.lower() == "json" else logging.Formatter( + "%(asctime)s %(levelname)s %(name)s — %(message)s" + ) + root = logging.getLogger() + for handler in root.handlers[:]: + root.removeHandler(handler) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + root.addHandler(handler) + root.setLevel(level) + + +class RequestContextMiddleware(BaseHTTPMiddleware): + """X-Request-ID + one access-log line per request.""" + + async def dispatch(self, request: Request, call_next) -> Response: + rid = request.headers.get("x-request-id") or uuid.uuid4().hex + token = request_id_var.set(rid) + started = time.perf_counter() + try: + response = await call_next(request) + except Exception: + duration_ms = round((time.perf_counter() - started) * 1000, 1) + logging.getLogger("procurement.access").exception( + "unhandled exception", + extra={"http_method": request.method, "http_path": request.url.path, + "http_status": 500, "duration_ms": duration_ms}, + ) + request_id_var.reset(token) + raise + duration_ms = round((time.perf_counter() - started) * 1000, 1) + response.headers["X-Request-ID"] = rid + request_id_var.reset(token) + if request.url.path not in _SKIP_ACCESS_LOG: + logging.getLogger("procurement.access").info( + "%s %s -> %s", + request.method, request.url.path, response.status_code, + extra={"http_method": request.method, "http_path": request.url.path, + "http_status": response.status_code, "duration_ms": duration_ms}, + ) + return response diff --git a/backend/app/routers/auth.py b/backend/app/routers/auth.py index 616810b..b021078 100644 --- a/backend/app/routers/auth.py +++ b/backend/app/routers/auth.py @@ -21,6 +21,7 @@ from app.auth import _verify_token, context_from_claims from app.config import settings from app.db.engine import get_db from app.routers.users import _user_projects, user_to_dict +from app.services import analytics from app.utils.errors import ServiceUnavailableError, UnauthorizedError logger = logging.getLogger("procurement.auth") @@ -75,6 +76,15 @@ async def login(payload: LoginRequest, db: AsyncSession = Depends(get_db)): claims = await _verify_token(access_token) ctx = await context_from_claims(claims, db) projects = await _user_projects(db, ctx.user) + await analytics.track( + "login_success", + {"username": payload.username, "role": ctx.role}, + profile_id=ctx.user.casdoor_id, + ) + await analytics.identify( + ctx.user.casdoor_id, + {"name": ctx.display_name, "role": ctx.role, "username": payload.username}, + ) return { "data": { "access_token": access_token, diff --git a/backend/app/routers/pm.py b/backend/app/routers/pm.py index be0be0f..a6ff13d 100644 --- a/backend/app/routers/pm.py +++ b/backend/app/routers/pm.py @@ -50,6 +50,7 @@ from app.models.procurement import ( Supplier, ) from app.schemas.procurement import PMReceiptCreate +from app.services import analytics from app.services.inventory_service import canonical_specs_hash from app.utils.errors import ( ForbiddenError, @@ -429,6 +430,16 @@ async def pm_confirm_receipt( quote.arrived_date = payload.received_at item_state = arrival_state(li.quantity, already_received + payload.received_qty) + await analytics.track( + "receipt_confirmed", + { + "receipt_id": str(receipt.id), + "quote_id": str(quote.id), + "quantity": str(payload.received_qty), + "state": item_state, + }, + profile_id=ctx.display_name, + ) return {"data": { "receipt_id": str(receipt.id), "state": item_state, diff --git a/backend/app/services/analytics.py b/backend/app/services/analytics.py new file mode 100644 index 0000000..be0b494 --- /dev/null +++ b/backend/app/services/analytics.py @@ -0,0 +1,76 @@ +"""Server-side product analytics — OpenPanel Track API. + +Fire-and-forget business events (quote lifecycle, receipts, logins) so the +OpenPanel dashboards show what actually happens inside the ERP, not just +page views. Never raises: analytics must not break a business flow. + +Configuration (all optional — disabled when any is missing): + PROCUREMENT_OPENPANEL_API_URL e.g. http://openpanel-api:3333 (self-hosted) + PROCUREMENT_OPENPANEL_CLIENT_ID project client id (write mode) + PROCUREMENT_OPENPANEL_CLIENT_SECRET project client secret + +Contract (OpenPanel Track API): POST {API_URL}/track with headers +``openpanel-client-id`` / ``openpanel-client-secret`` and a JSON body +``{"type": "track"|"identify", "profileId": ..., "event"/"properties": ...}``. +""" +import logging +from typing import Any + +import httpx + +from app.config import settings + +logger = logging.getLogger("procurement.analytics") + +_TIMEOUT = httpx.Timeout(3.0) + + +def _configured() -> bool: + return bool( + settings.OPENPANEL_API_URL + and settings.OPENPANEL_CLIENT_ID + and settings.OPENPANEL_CLIENT_SECRET + ) + + +async def track(event: str, properties: dict[str, Any] | None = None, profile_id: str | None = None) -> None: + """Record a named event; swallows every error.""" + if not _configured(): + return + body = { + "type": "track", + "event": event, + "properties": properties or {}, + } + if profile_id: + body["profileId"] = profile_id + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + await client.post( + f"{settings.OPENPANEL_API_URL.rstrip('/')}/track", + json=body, + headers={ + "openpanel-client-id": settings.OPENPANEL_CLIENT_ID, + "openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET, + }, + ) + except Exception as exc: # noqa: BLE001 — analytics must never raise + logger.debug("analytics track failed (%s): %s", event, exc) + + +async def identify(profile_id: str, properties: dict[str, Any] | None = None) -> None: + """Create/update a user profile (e.g. on login).""" + if not _configured(): + return + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + await client.post( + f"{settings.OPENPANEL_API_URL.rstrip('/')}/track", + json={"type": "identify", "profileId": profile_id, "properties": properties or {}}, + headers={ + "openpanel-client-id": settings.OPENPANEL_CLIENT_ID, + "openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET, + }, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("analytics identify failed: %s", exc) diff --git a/backend/app/services/quote_service.py b/backend/app/services/quote_service.py index 0cb0701..e469d89 100644 --- a/backend/app/services/quote_service.py +++ b/backend/app/services/quote_service.py @@ -4,6 +4,7 @@ Every mutation here runs in ONE ``async with db.begin()`` block; any failure rolls the whole operation back. All state changes log a quote_event in the same transaction. """ +import asyncio import logging from datetime import date, datetime, timezone from uuid import UUID @@ -33,6 +34,7 @@ from app.schemas.procurement import ( QuoteUpdate, RejectQuoteIn, ) +from app.services import analytics from app.services.material_service import ensure_brand_material_belong from app.utils.errors import ConflictError, NotFoundError, ValidationError @@ -56,6 +58,19 @@ def utcnow() -> datetime: def _add_event(db: AsyncSession, quote_id: UUID, event_type: str, payload: dict | None = None, actor: str | None = None) -> None: db.add(QuoteEvent(quote_id=quote_id, event_type=event_type, payload=payload or {}, actor=actor)) + # Product analytics (fire-and-forget, never blocks or raises): the whole + # quote lifecycle funnels through here, so one hook covers every + # status transition for the OpenPanel dashboards. + try: + asyncio.get_running_loop().create_task( + analytics.track( + f"quote_{event_type}", + {"quote_id": str(quote_id), "actor": actor, **(payload or {})}, + profile_id=actor, + ) + ) + except RuntimeError: # no running loop (defensive — callers are async) + pass async def get_quote_or_404(db: AsyncSession, quote_id: UUID) -> Quote: diff --git a/docker-compose.yml b/docker-compose.yml index 4dc4021..9970cff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,10 @@ services: ports: - "5433:5432" + networks: + - default + - shared # monitoring stack scrapes it (postgres-exporter) + volumes: - postgres_data:/var/lib/postgresql - ./sql/migration.sql:/docker-entrypoint-initdb.d/01-migration.sql:ro @@ -196,10 +200,28 @@ services: # Console admin used by POST /users/sync (identity mirroring). PROCUREMENT_CASDOOR_ADMIN_USER: admin PROCUREMENT_CASDOOR_ADMIN_PASSWORD: ${CASDOOR_ADMIN_PASSWORD:-123} + # Observability (Phase 2) — see monitoring/README.md + PROCUREMENT_LOG_FORMAT: ${LOG_FORMAT:-text} + PROCUREMENT_SENTRY_DSN: ${SENTRY_DSN_BACKEND:-} + PROCUREMENT_OPENPANEL_API_URL: ${OPENPANEL_API_URL:-} + PROCUREMENT_OPENPANEL_CLIENT_ID: ${OPENPANEL_CLIENT_ID:-} + PROCUREMENT_OPENPANEL_CLIENT_SECRET: ${OPENPANEL_CLIENT_SECRET:-} ports: - "8000:8000" + networks: + - traefik_network + - default + - shared # monitoring stack scrapes api:8000/metrics + + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request as u; u.urlopen('http://127.0.0.1:8000/health', timeout=3)\""] + interval: 30s + timeout: 5s + retries: 5 + start_period: 40s + labels: - traefik.enable=true - traefik.http.routers.api.rule=Host(`${API_HOST:-api.localhost}`) @@ -225,6 +247,10 @@ services: # Developer Mode panel on the login page — dev-only aid. Production # builds MUST leave this false (identity comes from Casdoor tokens). DEV_LOGIN_ENABLED: ${DEV_LOGIN_ENABLED:-false} + # Observability (build-time baked; empty = disabled) + VITE_OPENPANEL_URL: ${OPENPANEL_PUBLIC_URL:-} + VITE_OPENPANEL_CLIENT_ID: ${OPENPANEL_CLIENT_ID:-} + VITE_SENTRY_DSN: ${SENTRY_DSN_FRONTEND:-} container_name: procurement-frontend labels: @@ -248,6 +274,11 @@ networks: traefik_network: name: traefik_network driver: bridge + # Fixed-name network shared with the monitoring stack (monitoring compose + # joins it as external) — carries /metrics scrapes and postgres-exporter. + shared: + name: erp_shared + driver: bridge volumes: postgres_data: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 53cd9f6..9530fbb 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -19,6 +19,15 @@ RUN npm config set registry ${NPM_REGISTRY} && npm ci ARG DEV_LOGIN_ENABLED=false ENV VITE_ENABLE_DEV_LOGIN=${DEV_LOGIN_ENABLED} +# Observability (Phase 2) — baked into the bundle at build time. +# Empty values disable the feature (analytics.ts is a no-op then). +ARG VITE_OPENPANEL_URL="" +ARG VITE_OPENPANEL_CLIENT_ID="" +ARG VITE_SENTRY_DSN="" +ENV VITE_OPENPANEL_URL=${VITE_OPENPANEL_URL} \ + VITE_OPENPANEL_CLIENT_ID=${VITE_OPENPANEL_CLIENT_ID} \ + VITE_SENTRY_DSN=${VITE_SENTRY_DSN} + # Source code + build COPY . . RUN npm run build diff --git a/frontend/package.json b/frontend/package.json index 982b8fe..a3c5697 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,9 +10,11 @@ "preview": "vite preview" }, "dependencies": { + "@sentry/react": "^10.0.0", "@dnd-kit/core": "^6.3.1", "@dnd-kit/utilities": "^3.2.2", "@hookform/resolvers": "3.10", + "@openpanel/web": "^1.0.1", "@radix-ui/react-checkbox": "^1.3.11", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-label": "^2.1.15", diff --git a/frontend/src/components/ErrorBoundary.tsx b/frontend/src/components/ErrorBoundary.tsx index f263a55..6261fd6 100644 --- a/frontend/src/components/ErrorBoundary.tsx +++ b/frontend/src/components/ErrorBoundary.tsx @@ -1,4 +1,5 @@ import { Component, type ErrorInfo, type ReactNode } from "react"; +import { captureError } from "../lib/analytics"; interface Props { children: ReactNode; @@ -19,6 +20,8 @@ export class ErrorBoundary extends Component { componentDidCatch(error: Error, info: ErrorInfo) { console.error("ErrorBoundary caught:", error, info); + // GlitchTip/Sentry (no-op بدون VITE_SENTRY_DSN) + captureError(error); } private retry = () => { diff --git a/frontend/src/lib/analytics.ts b/frontend/src/lib/analytics.ts new file mode 100644 index 0000000..2362fc1 --- /dev/null +++ b/frontend/src/lib/analytics.ts @@ -0,0 +1,80 @@ +// ============================================================ +// Product analytics + error capture (Phase 2) +// +// - OpenPanel (@openpanel/web): funnels/usage — initialized only when +// VITE_OPENPANEL_URL and VITE_OPENPANEL_CLIENT_ID are set (build args), +// so local dev and air-gapped environments stay silent. +// - GlitchTip (@sentry/react): frontend exception capture, only when +// VITE_SENTRY_DSN is set. +// +// Both are no-ops without configuration; every helper is safe to call +// unconditionally from anywhere in the app. +// ============================================================ +import * as Sentry from "@sentry/react"; + +type Props = Record; + +declare global { + interface Window { + op?: (...args: unknown[]) => void; + } +} + +const OPENPANEL_URL = (import.meta.env?.VITE_OPENPANEL_URL as string | undefined)?.trim(); +const OPENPANEL_CLIENT_ID = (import.meta.env?.VITE_OPENPANEL_CLIENT_ID as string | undefined)?.trim(); +const SENTRY_DSN = (import.meta.env?.VITE_SENTRY_DSN as string | undefined)?.trim(); + +export function initAnalytics(): void { + if (SENTRY_DSN) { + Sentry.init({ + dsn: SENTRY_DSN, + environment: (import.meta.env?.MODE as string | undefined) ?? "production", + // Frontend traces are expensive and rarely needed — errors only. + tracesSampleRate: 0, + }); + } + if (OPENPANEL_URL && OPENPANEL_CLIENT_ID) { + try { + window.op?.("init", { + apiUrl: OPENPANEL_URL, + clientId: OPENPANEL_CLIENT_ID, + trackScreenViews: true, // pageview → which pages users visit + }); + } catch { + /* analytics must never break the app */ + } + } +} + +/** Attach the ERP user identity to analytics (after login / app load). */ +export function identifyUser(profile: { + id?: string; + name?: string; + role?: string | null; +}): void { + if (!profile.id) return; + try { + window.op?.("identify", { + profileId: profile.id, + firstName: profile.name || undefined, + data: { role: profile.role ?? undefined }, + }); + track("user_identified", { role: profile.role ?? undefined }); + } catch { + /* ignore */ + } +} + +export function track(event: string, properties?: Props): void { + try { + window.op?.("track", event, properties ?? {}); + } catch { + /* ignore */ + } +} + +export function captureError(err: unknown): void { + if (SENTRY_DSN) Sentry.captureException(err); +} + +export const analyticsEnabled = Boolean(OPENPANEL_URL && OPENPANEL_CLIENT_ID); diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 20e4b71..55d6bb6 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -6,6 +6,7 @@ import { Toaster } from "sonner"; import App from "./App"; import "./styles/globals.css"; import { queryClient } from "./lib/queryClient"; +import { initAnalytics } from "./lib/analytics"; // بعد از استقرار نسخه‌ی جدید، مرورگری که index.html قدیمی را باز کرده برای // بارگذاری chunkهای حذف‌شده 404 می‌گیرد و صفحه خالی می‌ماند. این شنونده، @@ -15,6 +16,9 @@ window.addEventListener("vite:preloadError", () => { window.location.reload(); }); +// Product analytics + GlitchTip error capture — no-ops without VITE_ config. +initAnalytics(); + createRoot(document.getElementById("root")!).render( diff --git a/frontend/src/pages/Auth/LoginPage.tsx b/frontend/src/pages/Auth/LoginPage.tsx index 548fcb8..9a78d68 100644 --- a/frontend/src/pages/Auth/LoginPage.tsx +++ b/frontend/src/pages/Auth/LoginPage.tsx @@ -3,6 +3,7 @@ import { useNavigate } from "react-router-dom"; import { toast } from "sonner"; import { ClipboardList, Eye, EyeOff, Loader2, Lock, ShieldCheck, Wrench, X } from "lucide-react"; import { api, apiV26 } from "../../lib/api"; +import { identifyUser, track } from "../../lib/analytics"; import { useAuthStore } from "../../store/auth"; import type { Project } from "../../lib/types"; @@ -78,6 +79,9 @@ export function LoginPage() { } setToken(access_token); setUser(user); + // تحلیل رفتار (no-op بدون تنظیمات VITE_OPENPANEL_*) + identifyUser({ id: user.id, name: user.display_name, role: user.role }); + track("login_success", { role: user.role }); toast.success(`خوش آمدید، ${user.display_name}`); // مدیر پروژه → پنل انبار؛ مدیر تدارکات/ادمین → دفتر پیشنهادهای خرید navigate(user.role === "pm" ? "/inventory" : "/"); diff --git a/frontend/src/pages/Wizard/WizardPage.tsx b/frontend/src/pages/Wizard/WizardPage.tsx index 6e573d0..bc1e2b4 100644 --- a/frontend/src/pages/Wizard/WizardPage.tsx +++ b/frontend/src/pages/Wizard/WizardPage.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useState } from "react"; +import { track } from "../../lib/analytics"; import { Link, useNavigate, useSearchParams } from "react-router-dom"; import { ChevronRight, ChevronLeft, Check } from "lucide-react"; import { toast } from "sonner"; @@ -118,6 +119,7 @@ export function WizardPage() { const goToStep = (n: number) => { setCurrentStep(n); setHighestVisited((h) => Math.max(h, n)); + track("wizard_step", { step: n, direction: n > currentStep ? "forward" : "back" }); window.scrollTo({ top: 0, behavior: "smooth" }); }; @@ -405,6 +407,7 @@ export function WizardPage() { } } toast.success("پیشنهاد ذخیره شد."); + track("quote_submitted", { quote_id: created.id, items: lineItems.length }); navigate("/"); } catch (e) { const target = resolveErrorTarget(e); diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts index 11f02fe..5b21142 100644 --- a/frontend/src/vite-env.d.ts +++ b/frontend/src/vite-env.d.ts @@ -1 +1,10 @@ /// + +// Observability build-time configuration (empty = feature disabled) +interface ImportMetaEnv { + readonly VITE_OPENPANEL_URL?: string; + readonly VITE_OPENPANEL_CLIENT_ID?: string; + readonly VITE_SENTRY_DSN?: string; + readonly VITE_ENABLE_DEV_LOGIN?: string; + readonly VITE_API_BASE_URL?: string; +} diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..4a7683f --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,111 @@ +# Monitoring & analytics stack + +What runs where, and why — the ERP gets the three missing observability +layers on the same 8 GB VPS: + +| Layer | Tool | What it answers | Budget | +|---|---|---|---| +| Product analytics | **OpenPanel** (self-hosted) | Which features are used? Where do users drop out of the quote wizard? Funnel: login → wizard steps → quote submitted → accepted → received. Session replay **off** by design (heavy; revisit with sampling later) | ~1–1.5 GB | +| Error tracking | **GlitchTip** (Sentry-compatible) | What broke, for whom, with which stack trace — frontend (ErrorBoundary + fetch errors) and backend (all unhandled exceptions) | ~750 MB | +| Metrics + logs | **Prometheus + Grafana + Loki/Promtail + node/postgres-exporters + cAdvisor** | Request rate/latency/5xx, container RAM, Postgres connections, disk pressure; full container logs searchable in Grafana | ~1.4 GB | + +The monitoring compose is a **separate compose project** — push-to-deploy +rebuilds of the app never restart monitoring containers. + +## First-time deployment (VPS) + +```bash +cd /srv/erp # the repo checkout used by the deploy webhook + +# 1) monitoring section of .env (see .env.example) — fill in real values: +# GLITCHTIP_SECRET_KEY, GLITCHTIP_DOMAIN/HOST, GLITCHTIP_DB_PASSWORD, +# GRAFANA_ADMIN_PASSWORD, POSTGRES_PASSWORD (already set), LOG_FORMAT=json + +# 2) the main compose gained a shared network — re-up once so it exists: +docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d + +# 3) start monitoring: +docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d + +# 4) GlitchTip is routed publicly through Traefik for the browser SDK: +docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env \ + --profile public up -d glitchtip +``` + +Admin UIs stay **loopback-only** (same posture as the MinIO console): + +```powershell +ssh -L 3001:127.0.0.1:3001 -L 8001:127.0.0.1:8001 @ +# Grafana → http://localhost:3001 (dashboards auto-provisioned) +# GlitchTip → http://localhost:8001 (create the first admin on first open) +``` + +In GlitchTip: create an **organization → project** for the backend and one +for the frontend, copy the DSNs into `/srv/erp/.env` +(`SENTRY_DSN_BACKEND`, `SENTRY_DSN_FRONTEND`) and redeploy the app once. + +## Product analytics (OpenPanel) + +Deployed from the upstream self-hosting branch (kept outside this repo — +it ships its own compose + update tooling): + +```bash +cd /opt +git clone -b self-hosting https://github.com/Openpanel-dev/openpanel.git +cd openpanel/self-hosting +./setup # interactive +./start +``` + +Then: + +1. Open the OpenPanel dashboard (expose it via Traefik on + `stats.bennu-api-bahrestan.ir` following its docs — add the container to + `traefik_network` and give the dashboard + its `/api` path labels), + create a project → copy the **client id / client secret**. +2. Put them in `/srv/erp/.env`: + `OPENPANEL_API_URL` (in-network URL of its api service), + `OPENPANEL_CLIENT_ID`, `OPENPANEL_CLIENT_SECRET`, and + `OPENPANEL_PUBLIC_URL` (browser-facing origin) — then rebuild the app: + `docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build api frontend`. + +What gets tracked automatically once configured: + +- **Frontend**: page views, `login_success`, `wizard_step` (funnel across + the 5 quote-wizard steps), `quote_submitted`, `user_identified`. +- **Backend** (server-side, cannot be blocked by the browser): + `login_success`, `quote_offered/edited/accepted/arrived/expired/rejected`, + `receipt_confirmed` — see `backend/app/services/analytics.py`. + +Without the env vars everything is a no-op — the app never waits on +analytics (3 s timeout, swallow-all). + +## Session replay + +Deliberately **off** (records full DOM of every session → storage + privacy +costs; low value for a small internal team at launch). If wanted later: +enable OpenPanel replay with **10–25% sampling** and review retention — +budget +1 GB storage and re-check RAM headroom first. + +## Alerting + +`monitoring/prometheus/alerts.yml` defines: API down, 5xx > 5%, p95 > 2 s, +root disk < 10%, host RAM < 10%, Postgres > 80 connections. They appear in +Grafana alerting; wire a notification channel (email/Telegram) from the +Grafana UI when ready. + +## Lean mode (if RAM gets tight) + +Drop prometheus+grafana+loki+promtail+cadvisor+node-exporter (~1.1 GB) and +keep GlitchTip + OpenPanel only; add `uptime-kuma` (~100 MB) for +up/down pings. The app keeps `/health/ready` either way — the deploy +webhook uses it post-rebuild regardless of the metrics stack. + +## What the app exposes for this stack + +- `GET /health` — liveness (process up). +- `GET /health/ready` — readiness: Postgres + MinIO + Casdoor checks. + Used by the deploy webhook after every rebuild. +- `GET /metrics` — Prometheus format (unauthenticated, for scraping). +- JSON request logs with `X-Request-ID` (set `LOG_FORMAT=json` in prod) + → shipped to Loki by Promtail. diff --git a/monitoring/docker-compose.monitoring.yml b/monitoring/docker-compose.monitoring.yml new file mode 100644 index 0000000..f28e7d0 --- /dev/null +++ b/monitoring/docker-compose.monitoring.yml @@ -0,0 +1,191 @@ +# ============================================================================ +# Monitoring stack — SEPARATE compose project from the ERP app on purpose: +# app rebuilds (push-to-deploy) must never restart monitoring containers. +# +# cd /srv/erp # repo root, so --env-file picks up the app's .env +# docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d +# +# Included (≈2.2 GB RAM budgeted): +# GlitchTip (errors, Sentry-compatible) + its own small Postgres +# Prometheus (metrics) + Grafana (dashboards) + Loki/Promtail (logs) +# node-exporter (host) + postgres-exporter (app DB) + cAdvisor (containers) +# +# Admin UIs (Grafana :3001, GlitchTip :8001) bind to 127.0.0.1 only — +# reach them via SSH tunnel exactly like the MinIO console. GlitchTip ALSO +# needs a public origin for the browser SDK: set GLITCHTIP_HOST and enable +# the "public" profile (Traefik route + TLS) — see monitoring/README.md. +# ============================================================================ + +services: + # ---------------------------------------------------------------- errors -- + glitchtip: + image: glitchtip/glitchtip:latest + container_name: monitoring-glitchtip + mem_limit: 640m + environment: + SECRET_KEY: ${GLITCHTIP_SECRET_KEY:?set GLITCHTIP_SECRET_KEY in .env} + GLITCHTIP_DOMAIN: ${GLITCHTIP_DOMAIN:?set GLITCHTIP_DOMAIN in .env} + DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASSWORD:-glitchtip}@glitchtip-postgres:5432/glitchtip + # Single-container mode (embedded worker) — no redis needed at this scale. + GLITCHTIP_EMBED_WORKER: "true" + VALKEY_URL: "" + ENABLE_USER_REGISTRATION: "false" + ENABLE_ORGANIZATION_CREATION: "false" + DEFAULT_FROM_EMAIL: ${GLITCHTIP_FROM_EMAIL:-noreply@bennu-api-bahrestan.ir} + EMAIL_URL: ${GLITCHTIP_EMAIL_URL:-consolemail://} + ports: + - "127.0.0.1:8001:8000" + depends_on: + glitchtip-postgres: + condition: service_healthy + networks: + - monitoring + - shared + - traefik # public DSN route (profile "public") + labels: + - traefik.enable=true + - traefik.http.routers.glitchtip.rule=Host(`${GLITCHTIP_HOST:-errors.localhost}`) + - traefik.http.routers.glitchtip.entrypoints=web + - traefik.http.services.glitchtip.loadbalancer.server.port=8000 + restart: unless-stopped + + glitchtip-postgres: + image: postgres:16-alpine + container_name: monitoring-glitchtip-postgres + mem_limit: 128m + environment: + POSTGRES_DB: glitchtip + POSTGRES_USER: glitchtip + POSTGRES_PASSWORD: ${GLITCHTIP_DB_PASSWORD:-glitchtip} + volumes: + - glitchtip_postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U glitchtip -d glitchtip"] + interval: 10s + timeout: 5s + retries: 10 + networks: + - monitoring + restart: unless-stopped + + # --------------------------------------------------------------- metrics -- + prometheus: + image: prom/prometheus:latest + container_name: monitoring-prometheus + mem_limit: 512m + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=14d + - --storage.tsdb.retention.size=2GB + volumes: + - ./prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + networks: + - monitoring + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + container_name: monitoring-grafana + mem_limit: 256m + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001/} + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "127.0.0.1:3001:3000" + networks: + - monitoring + restart: unless-stopped + + # ------------------------------------------------------------------ logs -- + loki: + image: grafana/loki:latest + container_name: monitoring-loki + mem_limit: 384m + command: -config.file=/etc/loki/loki-config.yml + volumes: + - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro + - loki_data:/loki + networks: + - monitoring + restart: unless-stopped + + promtail: + image: grafana/promtail:latest + container_name: monitoring-promtail + mem_limit: 64m + command: -config.file=/etc/promtail/promtail-config.yml + volumes: + - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - monitoring + restart: unless-stopped + + # -------------------------------------------------------------- exporters -- + node-exporter: + image: prom/node-exporter:latest + container_name: monitoring-node-exporter + mem_limit: 32m + command: + - --path.rootfs=/host + volumes: + - /:/host:ro,rslave + pid: host + networks: + - monitoring + restart: unless-stopped + + postgres-exporter: + image: prometheuscommunity/postgres-postgres-exporter:latest + container_name: monitoring-postgres-exporter + mem_limit: 48m + environment: + # App database on the main stack's postgres (reaches it via erp_shared). + DATA_SOURCE_NAME: postgres://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/procurement?sslmode=disable + networks: + - monitoring + - shared + restart: unless-stopped + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: monitoring-cadvisor + mem_limit: 128m + privileged: true + devices: + - /dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + networks: + - monitoring + restart: unless-stopped + +networks: + monitoring: + driver: bridge + # Reaches the ERP containers (api:8000/metrics, postgres:5432) — the main + # compose declares the same fixed-name network as "shared". + shared: + name: erp_shared + external: true + # Created by the main compose; required only for the public GlitchTip route. + traefik: + name: traefik_network + external: true + +volumes: + glitchtip_postgres_data: + prometheus_data: + grafana_data: + loki_data: diff --git a/monitoring/grafana/dashboards/erp-overview.json b/monitoring/grafana/dashboards/erp-overview.json new file mode 100644 index 0000000..85ec783 --- /dev/null +++ b/monitoring/grafana/dashboards/erp-overview.json @@ -0,0 +1,159 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (status) (rate(http_requests_total{job=\"erp-api\"}[5m]))", + "legendFormat": "HTTP {{status}}" + } + ], + "title": "API — request rate by status", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 10, "lineWidth": 2, "showPoints": "never" }, + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 2 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))", + "legendFormat": "p50" + } + ], + "title": "API — latency (p50 / p95)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 3, + "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (name) (container_memory_working_set_bytes{job=\"cadvisor\", name!=\"\", image!=\"\"})", + "legendFormat": "{{name}}" + } + ], + "title": "Containers — memory in use", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 4, + "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(pg_stat_activity_count)", + "legendFormat": "postgres connections" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "host RAM available" + } + ], + "title": "Database connections / host RAM", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, + "id": 5, + "options": { + "sortOrder": "Descending", + "wrapLogMessage": true, + "showTime": true, + "enableLogDetails": true + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "{compose_project=\"erp\"}", + "refId": "A" + } + ], + "title": "All ERP container logs (Loki)", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["erp"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "hide": 2, + "current": { "text": "Prometheus", "value": "Prometheus" } + }, + { + "name": "DS_LOKI", + "label": "Loki", + "type": "datasource", + "query": "loki", + "hide": 2, + "current": { "text": "Loki", "value": "Loki" } + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "ERP — Stack overview", + "uid": "erp-overview", + "version": 1 +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..fbfa660 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: erp-dashboards + orgId: 1 + folder: ERP + type: file + disableDeletion: false + updateIntervalSeconds: 60 + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/datasources.yml b/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..2c0808d --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: false diff --git a/monitoring/loki/loki-config.yml b/monitoring/loki/loki-config.yml new file mode 100644 index 0000000..096bdcf --- /dev/null +++ b/monitoring/loki/loki-config.yml @@ -0,0 +1,43 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + # Keep the log store bounded — this box shares 8 GB RAM / limited disk. + retention_period: 14d + ingestion_rate_mb: 4 + ingestion_burstsize_mb: 8 + max_query_series: 500 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +ruler: + storage: + type: local + local: + directory: /loki/rules diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..62858ac --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,58 @@ +# Alert rules surfaced in Grafana's alerting UI (no Alertmanager needed at +# this scale — Grafana can route them later via email/Telegram). +groups: + - name: erp-availability + rules: + - alert: ErpApiDown + expr: up{job="erp-api"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "ERP API is not scraping (down or /metrics failing)" + + - alert: ErpApiHighErrorRate + expr: | + sum(rate(http_requests_total{job="erp-api", status=~"5.."}[5m])) + / clamp_min(sum(rate(http_requests_total{job="erp-api"}[5m])), 0.001) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "ERP API 5xx rate above 5% for 5 minutes" + + - alert: ErpApiSlowRequests + expr: | + histogram_quantile(0.95, + sum(rate(http_request_duration_seconds_bucket{job="erp-api"}[5m])) by (le)) > 2 + for: 10m + labels: + severity: warning + annotations: + summary: "ERP API p95 latency above 2s for 10 minutes" + + - name: host + rules: + - alert: HostDiskAlmostFull + expr: (node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"}) < 0.1 + for: 15m + labels: + severity: critical + annotations: + summary: "Root filesystem below 10% free (deploy builds will fail)" + + - alert: HostMemoryPressure + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1 + for: 10m + labels: + severity: warning + annotations: + summary: "Host RAM below 10% available (check monitoring stack limits)" + + - alert: PostgresTooManyConnections + expr: sum(pg_stat_activity_count) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "PostgreSQL connection count above 80" diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..5418433 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,34 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + # The ERP API — /metrics is exposed by prometheus-fastapi-instrumentator + # (request rate, latency histogram, error counts). Reached over the + # erp_shared network the main compose also joins. + - job_name: erp-api + metrics_path: /metrics + static_configs: + - targets: ["api:8000"] + + # Host-level metrics (CPU, RAM, disk, network). + - job_name: node + static_configs: + - targets: ["node-exporter:9100"] + + # Per-container metrics (CPU, memory, restarts) for every container. + - job_name: cadvisor + static_configs: + - targets: ["cadvisor:8080"] + + # App database metrics (connections, cache hit ratio, WAL, ...). + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/monitoring/promtail/promtail-config.yml b/monitoring/promtail/promtail-config.yml new file mode 100644 index 0000000..e20584d --- /dev/null +++ b/monitoring/promtail/promtail-config.yml @@ -0,0 +1,40 @@ +# Ships every container's JSON logs to Loki (docker service discovery), +# tagging streams with the compose project / service / container name so +# logs are filterable in Grafana, e.g. {compose_service="api"}. +server: + http_listen_port: 9080 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 30s + filters: + - name: label + values: ["com.docker.compose.service"] + relabel_configs: + - source_labels: ["__meta_docker_container_label_com_docker_compose_project"] + target_label: compose_project + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + target_label: compose_service + - source_labels: ["__meta_docker_container_name"] + target_label: container + pipeline_stages: + - json: + expressions: + ts: time + stream: stream + log: log + - labels: + stream: + - timestamp: + source: ts + format: RFC3339Nano + - output: + source: log