feat(monitoring): observability stack + app instrumentation
- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
This commit is contained in:
MMD
+20
-2
@@ -40,9 +40,27 @@ DEV_LOGIN_ENABLED=false
|
||||
# --- Deploy webhook (Gitea push → rebuild; see infra/deploy-webhook/) ---
|
||||
GITEA_WEBHOOK_SECRET=CHANGE_ME_long_random_webhook_secret
|
||||
|
||||
# --- Monitoring stack (see monitoring/docker-compose.monitoring.yml) ---
|
||||
# --- Monitoring stack (see monitoring/README.md) ---
|
||||
# API log format for Loki ingestion ("text" locally, "json" in production)
|
||||
LOG_FORMAT=json
|
||||
# GlitchTip (error tracking). GLITCHTIP_HOST must be a public DNS name —
|
||||
# the browser SDK posts errors there.
|
||||
GLITCHTIP_SECRET_KEY=CHANGE_ME_random_django_secret
|
||||
GLITCHTIP_DOMAIN=https://errors.bennu-api-bahrestan.ir
|
||||
GLITCHTIP_HOST=errors.bennu-api-bahrestan.ir
|
||||
GLITCHTIP_DB_PASSWORD=CHANGE_ME_glitchtip_db_password
|
||||
GLITCHTIP_FROM_EMAIL=noreply@bennu-api-bahrestan.ir
|
||||
GLITCHTIP_EMAIL_URL=consolemail://
|
||||
# Grafana (dashboards) — UI is loopback-only (:3001) unless you add Traefik labels
|
||||
GRAFANA_ADMIN_PASSWORD=CHANGE_ME_grafana_password
|
||||
GRAFANA_ROOT_URL=http://localhost:3001/
|
||||
# OpenPanel (product analytics) — client id/secret from the OpenPanel project
|
||||
OPENPANEL_API_URL=http://openpanel-api:3333
|
||||
OPENPANEL_CLIENT_ID=
|
||||
OPENPANEL_CLIENT_SECRET=
|
||||
# Public origin of the OpenPanel dashboard/API the BROWSER talks to
|
||||
# (e.g. https://stats.bennu-api-bahrestan.ir — see monitoring/README.md)
|
||||
OPENPANEL_PUBLIC_URL=
|
||||
# Sentry/GlitchTip DSNs created in the GlitchTip UI after first boot
|
||||
SENTRY_DSN_BACKEND=
|
||||
SENTRY_DSN_FRONTEND=
|
||||
GRAFANA_ADMIN_PASSWORD=CHANGE_ME_grafana_password
|
||||
|
||||
@@ -78,6 +78,8 @@ class Settings(BaseSettings):
|
||||
INVENTORY_DEFAULT_PROJECT_ID: str = ""
|
||||
|
||||
# ---- Observability (Phase 2) -------------------------------------------
|
||||
# "text" (human console) | "json" (JSON lines for Loki/Promtail).
|
||||
LOG_FORMAT: str = "text"
|
||||
# Sentry/GlitchTip DSN — backend exception capture (empty = disabled).
|
||||
SENTRY_DSN: str = ""
|
||||
SENTRY_TRACES_SAMPLE_RATE: float = 0.0
|
||||
|
||||
+79
-1
@@ -7,12 +7,16 @@
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from prometheus_fastapi_instrumentator import Instrumentator
|
||||
from sqlalchemy import text as sql_text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.auth import get_current_user
|
||||
@@ -25,13 +29,29 @@ from app.db.init_db import (
|
||||
ensure_quote_projects,
|
||||
ensure_v26,
|
||||
)
|
||||
from app.observability import RequestContextMiddleware, setup_logging
|
||||
from app.routers import ROUTERS
|
||||
from app.services.quote_service import auto_expire_quotes
|
||||
from app.utils.errors import AppError
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s")
|
||||
setup_logging(settings.LOG_FORMAT)
|
||||
logger = logging.getLogger("procurement")
|
||||
|
||||
if settings.SENTRY_DSN:
|
||||
# GlitchTip/Sentry exception capture — after setup_logging so the SDK's
|
||||
# own logs go through our formatter too. Disabled entirely without a DSN.
|
||||
import sentry_sdk
|
||||
from sentry_sdk.integrations.fastapi import FastApiIntegration
|
||||
|
||||
sentry_sdk.init(
|
||||
dsn=settings.SENTRY_DSN,
|
||||
environment=settings.ENVIRONMENT,
|
||||
traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
|
||||
integrations=[FastApiIntegration()],
|
||||
send_default_pii=False,
|
||||
)
|
||||
logger.info("Sentry/GlitchTip exception capture enabled")
|
||||
|
||||
API_V1_PREFIX = "/api/v1"
|
||||
|
||||
|
||||
@@ -107,6 +127,12 @@ app.add_middleware(
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
|
||||
# Prometheus /metrics (requests, latency, errors) — mounted at app level,
|
||||
# outside the authenticated /api/v1 prefix; scrape target for the
|
||||
# monitoring stack.
|
||||
Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -203,3 +229,55 @@ for router in ROUTERS:
|
||||
@app.get("/health", tags=["system"])
|
||||
async def health():
|
||||
return {"status": "ok", "service": "procurement-api"}
|
||||
|
||||
|
||||
async def _check_db() -> bool:
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
await session.execute(sql_text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("readiness: database check failed")
|
||||
return False
|
||||
|
||||
|
||||
async def _check_minio() -> bool:
|
||||
try:
|
||||
from app.services.storage import get_client
|
||||
|
||||
get_client().bucket_exists(settings.MINIO_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("readiness: MinIO check failed")
|
||||
return False
|
||||
|
||||
|
||||
async def _check_casdoor() -> bool:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
resp = await client.get(
|
||||
settings.CASDOOR_JWKS_URL or f"{settings.CASDOOR_ENDPOINT}/.well-known/jwks"
|
||||
)
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
logger.exception("readiness: Casdoor JWKS check failed")
|
||||
return False
|
||||
|
||||
|
||||
@app.get("/health/ready", tags=["system"])
|
||||
async def health_ready():
|
||||
"""Readiness: verifies the dependencies the API actually needs.
|
||||
|
||||
The deploy webhook polls this after a rebuild, so a deploy that breaks
|
||||
the DB/MinIO/Casdoor connectivity is visible immediately.
|
||||
"""
|
||||
checks = {
|
||||
"database": await _check_db(),
|
||||
"minio": await _check_minio(),
|
||||
"casdoor": await _check_casdoor(),
|
||||
}
|
||||
ok = all(checks.values())
|
||||
return JSONResponse(
|
||||
status_code=200 if ok else 503,
|
||||
content={"status": "ok" if ok else "degraded", "service": "procurement-api", "checks": checks},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Observability plumbing — logging format, request IDs, request log.
|
||||
|
||||
- ``setup_logging()``: human-readable text by default, JSON lines when
|
||||
``PROCUREMENT_LOG_FORMAT=json`` (what Promtail/Loki prefers).
|
||||
- ``request_context`` middleware: assigns an X-Request-ID to every request
|
||||
(honours one sent by the edge), echoes it in the response, and logs one
|
||||
structured line per request (method, path, status, duration, request id)
|
||||
— this is what makes access logs useful in Loki.
|
||||
"""
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
request_id_var: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="-")
|
||||
|
||||
_SKIP_ACCESS_LOG = {"/health", "/health/ready", "/metrics"}
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Minimal JSON-lines formatter for Loki/Promtail ingestion."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = {
|
||||
"ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
"request_id": request_id_var.get(),
|
||||
}
|
||||
if record.exc_info:
|
||||
payload["exception"] = self.formatException(record.exc_info)
|
||||
for key in ("http_method", "http_path", "http_status", "duration_ms"):
|
||||
value = getattr(record, key, None)
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def setup_logging(log_format: str = "text", level: int = logging.INFO) -> None:
|
||||
formatter = JsonFormatter() if log_format.lower() == "json" else logging.Formatter(
|
||||
"%(asctime)s %(levelname)s %(name)s — %(message)s"
|
||||
)
|
||||
root = logging.getLogger()
|
||||
for handler in root.handlers[:]:
|
||||
root.removeHandler(handler)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
root.addHandler(handler)
|
||||
root.setLevel(level)
|
||||
|
||||
|
||||
class RequestContextMiddleware(BaseHTTPMiddleware):
|
||||
"""X-Request-ID + one access-log line per request."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
rid = request.headers.get("x-request-id") or uuid.uuid4().hex
|
||||
token = request_id_var.set(rid)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
logging.getLogger("procurement.access").exception(
|
||||
"unhandled exception",
|
||||
extra={"http_method": request.method, "http_path": request.url.path,
|
||||
"http_status": 500, "duration_ms": duration_ms},
|
||||
)
|
||||
request_id_var.reset(token)
|
||||
raise
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
response.headers["X-Request-ID"] = rid
|
||||
request_id_var.reset(token)
|
||||
if request.url.path not in _SKIP_ACCESS_LOG:
|
||||
logging.getLogger("procurement.access").info(
|
||||
"%s %s -> %s",
|
||||
request.method, request.url.path, response.status_code,
|
||||
extra={"http_method": request.method, "http_path": request.url.path,
|
||||
"http_status": response.status_code, "duration_ms": duration_ms},
|
||||
)
|
||||
return response
|
||||
@@ -21,6 +21,7 @@ from app.auth import _verify_token, context_from_claims
|
||||
from app.config import settings
|
||||
from app.db.engine import get_db
|
||||
from app.routers.users import _user_projects, user_to_dict
|
||||
from app.services import analytics
|
||||
from app.utils.errors import ServiceUnavailableError, UnauthorizedError
|
||||
|
||||
logger = logging.getLogger("procurement.auth")
|
||||
@@ -75,6 +76,15 @@ async def login(payload: LoginRequest, db: AsyncSession = Depends(get_db)):
|
||||
claims = await _verify_token(access_token)
|
||||
ctx = await context_from_claims(claims, db)
|
||||
projects = await _user_projects(db, ctx.user)
|
||||
await analytics.track(
|
||||
"login_success",
|
||||
{"username": payload.username, "role": ctx.role},
|
||||
profile_id=ctx.user.casdoor_id,
|
||||
)
|
||||
await analytics.identify(
|
||||
ctx.user.casdoor_id,
|
||||
{"name": ctx.display_name, "role": ctx.role, "username": payload.username},
|
||||
)
|
||||
return {
|
||||
"data": {
|
||||
"access_token": access_token,
|
||||
|
||||
@@ -50,6 +50,7 @@ from app.models.procurement import (
|
||||
Supplier,
|
||||
)
|
||||
from app.schemas.procurement import PMReceiptCreate
|
||||
from app.services import analytics
|
||||
from app.services.inventory_service import canonical_specs_hash
|
||||
from app.utils.errors import (
|
||||
ForbiddenError,
|
||||
@@ -429,6 +430,16 @@ async def pm_confirm_receipt(
|
||||
quote.arrived_date = payload.received_at
|
||||
|
||||
item_state = arrival_state(li.quantity, already_received + payload.received_qty)
|
||||
await analytics.track(
|
||||
"receipt_confirmed",
|
||||
{
|
||||
"receipt_id": str(receipt.id),
|
||||
"quote_id": str(quote.id),
|
||||
"quantity": str(payload.received_qty),
|
||||
"state": item_state,
|
||||
},
|
||||
profile_id=ctx.display_name,
|
||||
)
|
||||
return {"data": {
|
||||
"receipt_id": str(receipt.id),
|
||||
"state": item_state,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Server-side product analytics — OpenPanel Track API.
|
||||
|
||||
Fire-and-forget business events (quote lifecycle, receipts, logins) so the
|
||||
OpenPanel dashboards show what actually happens inside the ERP, not just
|
||||
page views. Never raises: analytics must not break a business flow.
|
||||
|
||||
Configuration (all optional — disabled when any is missing):
|
||||
PROCUREMENT_OPENPANEL_API_URL e.g. http://openpanel-api:3333 (self-hosted)
|
||||
PROCUREMENT_OPENPANEL_CLIENT_ID project client id (write mode)
|
||||
PROCUREMENT_OPENPANEL_CLIENT_SECRET project client secret
|
||||
|
||||
Contract (OpenPanel Track API): POST {API_URL}/track with headers
|
||||
``openpanel-client-id`` / ``openpanel-client-secret`` and a JSON body
|
||||
``{"type": "track"|"identify", "profileId": ..., "event"/"properties": ...}``.
|
||||
"""
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger("procurement.analytics")
|
||||
|
||||
_TIMEOUT = httpx.Timeout(3.0)
|
||||
|
||||
|
||||
def _configured() -> bool:
|
||||
return bool(
|
||||
settings.OPENPANEL_API_URL
|
||||
and settings.OPENPANEL_CLIENT_ID
|
||||
and settings.OPENPANEL_CLIENT_SECRET
|
||||
)
|
||||
|
||||
|
||||
async def track(event: str, properties: dict[str, Any] | None = None, profile_id: str | None = None) -> None:
|
||||
"""Record a named event; swallows every error."""
|
||||
if not _configured():
|
||||
return
|
||||
body = {
|
||||
"type": "track",
|
||||
"event": event,
|
||||
"properties": properties or {},
|
||||
}
|
||||
if profile_id:
|
||||
body["profileId"] = profile_id
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
await client.post(
|
||||
f"{settings.OPENPANEL_API_URL.rstrip('/')}/track",
|
||||
json=body,
|
||||
headers={
|
||||
"openpanel-client-id": settings.OPENPANEL_CLIENT_ID,
|
||||
"openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — analytics must never raise
|
||||
logger.debug("analytics track failed (%s): %s", event, exc)
|
||||
|
||||
|
||||
async def identify(profile_id: str, properties: dict[str, Any] | None = None) -> None:
|
||||
"""Create/update a user profile (e.g. on login)."""
|
||||
if not _configured():
|
||||
return
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
await client.post(
|
||||
f"{settings.OPENPANEL_API_URL.rstrip('/')}/track",
|
||||
json={"type": "identify", "profileId": profile_id, "properties": properties or {}},
|
||||
headers={
|
||||
"openpanel-client-id": settings.OPENPANEL_CLIENT_ID,
|
||||
"openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("analytics identify failed: %s", exc)
|
||||
@@ -4,6 +4,7 @@ Every mutation here runs in ONE ``async with db.begin()`` block; any failure
|
||||
rolls the whole operation back. All state changes log a quote_event in the
|
||||
same transaction.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date, datetime, timezone
|
||||
from uuid import UUID
|
||||
@@ -33,6 +34,7 @@ from app.schemas.procurement import (
|
||||
QuoteUpdate,
|
||||
RejectQuoteIn,
|
||||
)
|
||||
from app.services import analytics
|
||||
from app.services.material_service import ensure_brand_material_belong
|
||||
from app.utils.errors import ConflictError, NotFoundError, ValidationError
|
||||
|
||||
@@ -56,6 +58,19 @@ def utcnow() -> datetime:
|
||||
|
||||
def _add_event(db: AsyncSession, quote_id: UUID, event_type: str, payload: dict | None = None, actor: str | None = None) -> None:
|
||||
db.add(QuoteEvent(quote_id=quote_id, event_type=event_type, payload=payload or {}, actor=actor))
|
||||
# Product analytics (fire-and-forget, never blocks or raises): the whole
|
||||
# quote lifecycle funnels through here, so one hook covers every
|
||||
# status transition for the OpenPanel dashboards.
|
||||
try:
|
||||
asyncio.get_running_loop().create_task(
|
||||
analytics.track(
|
||||
f"quote_{event_type}",
|
||||
{"quote_id": str(quote_id), "actor": actor, **(payload or {})},
|
||||
profile_id=actor,
|
||||
)
|
||||
)
|
||||
except RuntimeError: # no running loop (defensive — callers are async)
|
||||
pass
|
||||
|
||||
|
||||
async def get_quote_or_404(db: AsyncSession, quote_id: UUID) -> Quote:
|
||||
|
||||
@@ -13,6 +13,10 @@ services:
|
||||
ports:
|
||||
- "5433:5432"
|
||||
|
||||
networks:
|
||||
- default
|
||||
- shared # monitoring stack scrapes it (postgres-exporter)
|
||||
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
- ./sql/migration.sql:/docker-entrypoint-initdb.d/01-migration.sql:ro
|
||||
@@ -196,10 +200,28 @@ services:
|
||||
# Console admin used by POST /users/sync (identity mirroring).
|
||||
PROCUREMENT_CASDOOR_ADMIN_USER: admin
|
||||
PROCUREMENT_CASDOOR_ADMIN_PASSWORD: ${CASDOOR_ADMIN_PASSWORD:-123}
|
||||
# Observability (Phase 2) — see monitoring/README.md
|
||||
PROCUREMENT_LOG_FORMAT: ${LOG_FORMAT:-text}
|
||||
PROCUREMENT_SENTRY_DSN: ${SENTRY_DSN_BACKEND:-}
|
||||
PROCUREMENT_OPENPANEL_API_URL: ${OPENPANEL_API_URL:-}
|
||||
PROCUREMENT_OPENPANEL_CLIENT_ID: ${OPENPANEL_CLIENT_ID:-}
|
||||
PROCUREMENT_OPENPANEL_CLIENT_SECRET: ${OPENPANEL_CLIENT_SECRET:-}
|
||||
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
||||
networks:
|
||||
- traefik_network
|
||||
- default
|
||||
- shared # monitoring stack scrapes api:8000/metrics
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request as u; u.urlopen('http://127.0.0.1:8000/health', timeout=3)\""]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 40s
|
||||
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.api.rule=Host(`${API_HOST:-api.localhost}`)
|
||||
@@ -225,6 +247,10 @@ services:
|
||||
# Developer Mode panel on the login page — dev-only aid. Production
|
||||
# builds MUST leave this false (identity comes from Casdoor tokens).
|
||||
DEV_LOGIN_ENABLED: ${DEV_LOGIN_ENABLED:-false}
|
||||
# Observability (build-time baked; empty = disabled)
|
||||
VITE_OPENPANEL_URL: ${OPENPANEL_PUBLIC_URL:-}
|
||||
VITE_OPENPANEL_CLIENT_ID: ${OPENPANEL_CLIENT_ID:-}
|
||||
VITE_SENTRY_DSN: ${SENTRY_DSN_FRONTEND:-}
|
||||
container_name: procurement-frontend
|
||||
|
||||
labels:
|
||||
@@ -248,6 +274,11 @@ networks:
|
||||
traefik_network:
|
||||
name: traefik_network
|
||||
driver: bridge
|
||||
# Fixed-name network shared with the monitoring stack (monitoring compose
|
||||
# joins it as external) — carries /metrics scrapes and postgres-exporter.
|
||||
shared:
|
||||
name: erp_shared
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
|
||||
@@ -19,6 +19,15 @@ RUN npm config set registry ${NPM_REGISTRY} && npm ci
|
||||
ARG DEV_LOGIN_ENABLED=false
|
||||
ENV VITE_ENABLE_DEV_LOGIN=${DEV_LOGIN_ENABLED}
|
||||
|
||||
# Observability (Phase 2) — baked into the bundle at build time.
|
||||
# Empty values disable the feature (analytics.ts is a no-op then).
|
||||
ARG VITE_OPENPANEL_URL=""
|
||||
ARG VITE_OPENPANEL_CLIENT_ID=""
|
||||
ARG VITE_SENTRY_DSN=""
|
||||
ENV VITE_OPENPANEL_URL=${VITE_OPENPANEL_URL} \
|
||||
VITE_OPENPANEL_CLIENT_ID=${VITE_OPENPANEL_CLIENT_ID} \
|
||||
VITE_SENTRY_DSN=${VITE_SENTRY_DSN}
|
||||
|
||||
# Source code + build
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
@@ -10,9 +10,11 @@
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sentry/react": "^10.0.0",
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@hookform/resolvers": "3.10",
|
||||
"@openpanel/web": "^1.0.1",
|
||||
"@radix-ui/react-checkbox": "^1.3.11",
|
||||
"@radix-ui/react-dialog": "^1.1.23",
|
||||
"@radix-ui/react-label": "^2.1.15",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Component, type ErrorInfo, type ReactNode } from "react";
|
||||
import { captureError } from "../lib/analytics";
|
||||
|
||||
interface Props {
|
||||
children: ReactNode;
|
||||
@@ -19,6 +20,8 @@ export class ErrorBoundary extends Component<Props, State> {
|
||||
|
||||
componentDidCatch(error: Error, info: ErrorInfo) {
|
||||
console.error("ErrorBoundary caught:", error, info);
|
||||
// GlitchTip/Sentry (no-op بدون VITE_SENTRY_DSN)
|
||||
captureError(error);
|
||||
}
|
||||
|
||||
private retry = () => {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// ============================================================
|
||||
// Product analytics + error capture (Phase 2)
|
||||
//
|
||||
// - OpenPanel (@openpanel/web): funnels/usage — initialized only when
|
||||
// VITE_OPENPANEL_URL and VITE_OPENPANEL_CLIENT_ID are set (build args),
|
||||
// so local dev and air-gapped environments stay silent.
|
||||
// - GlitchTip (@sentry/react): frontend exception capture, only when
|
||||
// VITE_SENTRY_DSN is set.
|
||||
//
|
||||
// Both are no-ops without configuration; every helper is safe to call
|
||||
// unconditionally from anywhere in the app.
|
||||
// ============================================================
|
||||
import * as Sentry from "@sentry/react";
|
||||
|
||||
type Props = Record<string, unknown>;
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
op?: (...args: unknown[]) => void;
|
||||
}
|
||||
}
|
||||
|
||||
const OPENPANEL_URL = (import.meta.env?.VITE_OPENPANEL_URL as string | undefined)?.trim();
|
||||
const OPENPANEL_CLIENT_ID = (import.meta.env?.VITE_OPENPANEL_CLIENT_ID as string | undefined)?.trim();
|
||||
const SENTRY_DSN = (import.meta.env?.VITE_SENTRY_DSN as string | undefined)?.trim();
|
||||
|
||||
export function initAnalytics(): void {
|
||||
if (SENTRY_DSN) {
|
||||
Sentry.init({
|
||||
dsn: SENTRY_DSN,
|
||||
environment: (import.meta.env?.MODE as string | undefined) ?? "production",
|
||||
// Frontend traces are expensive and rarely needed — errors only.
|
||||
tracesSampleRate: 0,
|
||||
});
|
||||
}
|
||||
if (OPENPANEL_URL && OPENPANEL_CLIENT_ID) {
|
||||
try {
|
||||
window.op?.("init", {
|
||||
apiUrl: OPENPANEL_URL,
|
||||
clientId: OPENPANEL_CLIENT_ID,
|
||||
trackScreenViews: true, // pageview → which pages users visit
|
||||
});
|
||||
} catch {
|
||||
/* analytics must never break the app */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Attach the ERP user identity to analytics (after login / app load). */
|
||||
export function identifyUser(profile: {
|
||||
id?: string;
|
||||
name?: string;
|
||||
role?: string | null;
|
||||
}): void {
|
||||
if (!profile.id) return;
|
||||
try {
|
||||
window.op?.("identify", {
|
||||
profileId: profile.id,
|
||||
firstName: profile.name || undefined,
|
||||
data: { role: profile.role ?? undefined },
|
||||
});
|
||||
track("user_identified", { role: profile.role ?? undefined });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
export function track(event: string, properties?: Props): void {
|
||||
try {
|
||||
window.op?.("track", event, properties ?? {});
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
export function captureError(err: unknown): void {
|
||||
if (SENTRY_DSN) Sentry.captureException(err);
|
||||
}
|
||||
|
||||
export const analyticsEnabled = Boolean(OPENPANEL_URL && OPENPANEL_CLIENT_ID);
|
||||
@@ -6,6 +6,7 @@ import { Toaster } from "sonner";
|
||||
import App from "./App";
|
||||
import "./styles/globals.css";
|
||||
import { queryClient } from "./lib/queryClient";
|
||||
import { initAnalytics } from "./lib/analytics";
|
||||
|
||||
// بعد از استقرار نسخهی جدید، مرورگری که index.html قدیمی را باز کرده برای
|
||||
// بارگذاری chunkهای حذفشده 404 میگیرد و صفحه خالی میماند. این شنونده،
|
||||
@@ -15,6 +16,9 @@ window.addEventListener("vite:preloadError", () => {
|
||||
window.location.reload();
|
||||
});
|
||||
|
||||
// Product analytics + GlitchTip error capture — no-ops without VITE_ config.
|
||||
initAnalytics();
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<StrictMode>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useNavigate } from "react-router-dom";
|
||||
import { toast } from "sonner";
|
||||
import { ClipboardList, Eye, EyeOff, Loader2, Lock, ShieldCheck, Wrench, X } from "lucide-react";
|
||||
import { api, apiV26 } from "../../lib/api";
|
||||
import { identifyUser, track } from "../../lib/analytics";
|
||||
import { useAuthStore } from "../../store/auth";
|
||||
import type { Project } from "../../lib/types";
|
||||
|
||||
@@ -78,6 +79,9 @@ export function LoginPage() {
|
||||
}
|
||||
setToken(access_token);
|
||||
setUser(user);
|
||||
// تحلیل رفتار (no-op بدون تنظیمات VITE_OPENPANEL_*)
|
||||
identifyUser({ id: user.id, name: user.display_name, role: user.role });
|
||||
track("login_success", { role: user.role });
|
||||
toast.success(`خوش آمدید، ${user.display_name}`);
|
||||
// مدیر پروژه → پنل انبار؛ مدیر تدارکات/ادمین → دفتر پیشنهادهای خرید
|
||||
navigate(user.role === "pm" ? "/inventory" : "/");
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { track } from "../../lib/analytics";
|
||||
import { Link, useNavigate, useSearchParams } from "react-router-dom";
|
||||
import { ChevronRight, ChevronLeft, Check } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
@@ -118,6 +119,7 @@ export function WizardPage() {
|
||||
const goToStep = (n: number) => {
|
||||
setCurrentStep(n);
|
||||
setHighestVisited((h) => Math.max(h, n));
|
||||
track("wizard_step", { step: n, direction: n > currentStep ? "forward" : "back" });
|
||||
window.scrollTo({ top: 0, behavior: "smooth" });
|
||||
};
|
||||
|
||||
@@ -405,6 +407,7 @@ export function WizardPage() {
|
||||
}
|
||||
}
|
||||
toast.success("پیشنهاد ذخیره شد.");
|
||||
track("quote_submitted", { quote_id: created.id, items: lineItems.length });
|
||||
navigate("/");
|
||||
} catch (e) {
|
||||
const target = resolveErrorTarget(e);
|
||||
|
||||
Vendored
+9
@@ -1 +1,10 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
// Observability build-time configuration (empty = feature disabled)
|
||||
interface ImportMetaEnv {
|
||||
readonly VITE_OPENPANEL_URL?: string;
|
||||
readonly VITE_OPENPANEL_CLIENT_ID?: string;
|
||||
readonly VITE_SENTRY_DSN?: string;
|
||||
readonly VITE_ENABLE_DEV_LOGIN?: string;
|
||||
readonly VITE_API_BASE_URL?: string;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# Monitoring & analytics stack
|
||||
|
||||
What runs where, and why — the ERP gets the three missing observability
|
||||
layers on the same 8 GB VPS:
|
||||
|
||||
| Layer | Tool | What it answers | Budget |
|
||||
|---|---|---|---|
|
||||
| Product analytics | **OpenPanel** (self-hosted) | Which features are used? Where do users drop out of the quote wizard? Funnel: login → wizard steps → quote submitted → accepted → received. Session replay **off** by design (heavy; revisit with sampling later) | ~1–1.5 GB |
|
||||
| Error tracking | **GlitchTip** (Sentry-compatible) | What broke, for whom, with which stack trace — frontend (ErrorBoundary + fetch errors) and backend (all unhandled exceptions) | ~750 MB |
|
||||
| Metrics + logs | **Prometheus + Grafana + Loki/Promtail + node/postgres-exporters + cAdvisor** | Request rate/latency/5xx, container RAM, Postgres connections, disk pressure; full container logs searchable in Grafana | ~1.4 GB |
|
||||
|
||||
The monitoring compose is a **separate compose project** — push-to-deploy
|
||||
rebuilds of the app never restart monitoring containers.
|
||||
|
||||
## First-time deployment (VPS)
|
||||
|
||||
```bash
|
||||
cd /srv/erp # the repo checkout used by the deploy webhook
|
||||
|
||||
# 1) monitoring section of .env (see .env.example) — fill in real values:
|
||||
# GLITCHTIP_SECRET_KEY, GLITCHTIP_DOMAIN/HOST, GLITCHTIP_DB_PASSWORD,
|
||||
# GRAFANA_ADMIN_PASSWORD, POSTGRES_PASSWORD (already set), LOG_FORMAT=json
|
||||
|
||||
# 2) the main compose gained a shared network — re-up once so it exists:
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
|
||||
# 3) start monitoring:
|
||||
docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d
|
||||
|
||||
# 4) GlitchTip is routed publicly through Traefik for the browser SDK:
|
||||
docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env \
|
||||
--profile public up -d glitchtip
|
||||
```
|
||||
|
||||
Admin UIs stay **loopback-only** (same posture as the MinIO console):
|
||||
|
||||
```powershell
|
||||
ssh -L 3001:127.0.0.1:3001 -L 8001:127.0.0.1:8001 <user>@<server-host>
|
||||
# Grafana → http://localhost:3001 (dashboards auto-provisioned)
|
||||
# GlitchTip → http://localhost:8001 (create the first admin on first open)
|
||||
```
|
||||
|
||||
In GlitchTip: create an **organization → project** for the backend and one
|
||||
for the frontend, copy the DSNs into `/srv/erp/.env`
|
||||
(`SENTRY_DSN_BACKEND`, `SENTRY_DSN_FRONTEND`) and redeploy the app once.
|
||||
|
||||
## Product analytics (OpenPanel)
|
||||
|
||||
Deployed from the upstream self-hosting branch (kept outside this repo —
|
||||
it ships its own compose + update tooling):
|
||||
|
||||
```bash
|
||||
cd /opt
|
||||
git clone -b self-hosting https://github.com/Openpanel-dev/openpanel.git
|
||||
cd openpanel/self-hosting
|
||||
./setup # interactive
|
||||
./start
|
||||
```
|
||||
|
||||
Then:
|
||||
|
||||
1. Open the OpenPanel dashboard (expose it via Traefik on
|
||||
`stats.bennu-api-bahrestan.ir` following its docs — add the container to
|
||||
`traefik_network` and give the dashboard + its `/api` path labels),
|
||||
create a project → copy the **client id / client secret**.
|
||||
2. Put them in `/srv/erp/.env`:
|
||||
`OPENPANEL_API_URL` (in-network URL of its api service),
|
||||
`OPENPANEL_CLIENT_ID`, `OPENPANEL_CLIENT_SECRET`, and
|
||||
`OPENPANEL_PUBLIC_URL` (browser-facing origin) — then rebuild the app:
|
||||
`docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build api frontend`.
|
||||
|
||||
What gets tracked automatically once configured:
|
||||
|
||||
- **Frontend**: page views, `login_success`, `wizard_step` (funnel across
|
||||
the 5 quote-wizard steps), `quote_submitted`, `user_identified`.
|
||||
- **Backend** (server-side, cannot be blocked by the browser):
|
||||
`login_success`, `quote_offered/edited/accepted/arrived/expired/rejected`,
|
||||
`receipt_confirmed` — see `backend/app/services/analytics.py`.
|
||||
|
||||
Without the env vars everything is a no-op — the app never waits on
|
||||
analytics (3 s timeout, swallow-all).
|
||||
|
||||
## Session replay
|
||||
|
||||
Deliberately **off** (records full DOM of every session → storage + privacy
|
||||
costs; low value for a small internal team at launch). If wanted later:
|
||||
enable OpenPanel replay with **10–25% sampling** and review retention —
|
||||
budget +1 GB storage and re-check RAM headroom first.
|
||||
|
||||
## Alerting
|
||||
|
||||
`monitoring/prometheus/alerts.yml` defines: API down, 5xx > 5%, p95 > 2 s,
|
||||
root disk < 10%, host RAM < 10%, Postgres > 80 connections. They appear in
|
||||
Grafana alerting; wire a notification channel (email/Telegram) from the
|
||||
Grafana UI when ready.
|
||||
|
||||
## Lean mode (if RAM gets tight)
|
||||
|
||||
Drop prometheus+grafana+loki+promtail+cadvisor+node-exporter (~1.1 GB) and
|
||||
keep GlitchTip + OpenPanel only; add `uptime-kuma` (~100 MB) for
|
||||
up/down pings. The app keeps `/health/ready` either way — the deploy
|
||||
webhook uses it post-rebuild regardless of the metrics stack.
|
||||
|
||||
## What the app exposes for this stack
|
||||
|
||||
- `GET /health` — liveness (process up).
|
||||
- `GET /health/ready` — readiness: Postgres + MinIO + Casdoor checks.
|
||||
Used by the deploy webhook after every rebuild.
|
||||
- `GET /metrics` — Prometheus format (unauthenticated, for scraping).
|
||||
- JSON request logs with `X-Request-ID` (set `LOG_FORMAT=json` in prod)
|
||||
→ shipped to Loki by Promtail.
|
||||
@@ -0,0 +1,191 @@
|
||||
# ============================================================================
|
||||
# Monitoring stack — SEPARATE compose project from the ERP app on purpose:
|
||||
# app rebuilds (push-to-deploy) must never restart monitoring containers.
|
||||
#
|
||||
# cd /srv/erp # repo root, so --env-file picks up the app's .env
|
||||
# docker compose -f monitoring/docker-compose.monitoring.yml --env-file .env up -d
|
||||
#
|
||||
# Included (≈2.2 GB RAM budgeted):
|
||||
# GlitchTip (errors, Sentry-compatible) + its own small Postgres
|
||||
# Prometheus (metrics) + Grafana (dashboards) + Loki/Promtail (logs)
|
||||
# node-exporter (host) + postgres-exporter (app DB) + cAdvisor (containers)
|
||||
#
|
||||
# Admin UIs (Grafana :3001, GlitchTip :8001) bind to 127.0.0.1 only —
|
||||
# reach them via SSH tunnel exactly like the MinIO console. GlitchTip ALSO
|
||||
# needs a public origin for the browser SDK: set GLITCHTIP_HOST and enable
|
||||
# the "public" profile (Traefik route + TLS) — see monitoring/README.md.
|
||||
# ============================================================================
|
||||
|
||||
services:
|
||||
# ---------------------------------------------------------------- errors --
|
||||
glitchtip:
|
||||
image: glitchtip/glitchtip:latest
|
||||
container_name: monitoring-glitchtip
|
||||
mem_limit: 640m
|
||||
environment:
|
||||
SECRET_KEY: ${GLITCHTIP_SECRET_KEY:?set GLITCHTIP_SECRET_KEY in .env}
|
||||
GLITCHTIP_DOMAIN: ${GLITCHTIP_DOMAIN:?set GLITCHTIP_DOMAIN in .env}
|
||||
DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASSWORD:-glitchtip}@glitchtip-postgres:5432/glitchtip
|
||||
# Single-container mode (embedded worker) — no redis needed at this scale.
|
||||
GLITCHTIP_EMBED_WORKER: "true"
|
||||
VALKEY_URL: ""
|
||||
ENABLE_USER_REGISTRATION: "false"
|
||||
ENABLE_ORGANIZATION_CREATION: "false"
|
||||
DEFAULT_FROM_EMAIL: ${GLITCHTIP_FROM_EMAIL:-noreply@bennu-api-bahrestan.ir}
|
||||
EMAIL_URL: ${GLITCHTIP_EMAIL_URL:-consolemail://}
|
||||
ports:
|
||||
- "127.0.0.1:8001:8000"
|
||||
depends_on:
|
||||
glitchtip-postgres:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- monitoring
|
||||
- shared
|
||||
- traefik # public DSN route (profile "public")
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.glitchtip.rule=Host(`${GLITCHTIP_HOST:-errors.localhost}`)
|
||||
- traefik.http.routers.glitchtip.entrypoints=web
|
||||
- traefik.http.services.glitchtip.loadbalancer.server.port=8000
|
||||
restart: unless-stopped
|
||||
|
||||
glitchtip-postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: monitoring-glitchtip-postgres
|
||||
mem_limit: 128m
|
||||
environment:
|
||||
POSTGRES_DB: glitchtip
|
||||
POSTGRES_USER: glitchtip
|
||||
POSTGRES_PASSWORD: ${GLITCHTIP_DB_PASSWORD:-glitchtip}
|
||||
volumes:
|
||||
- glitchtip_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U glitchtip -d glitchtip"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
# --------------------------------------------------------------- metrics --
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
container_name: monitoring-prometheus
|
||||
mem_limit: 512m
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.retention.time=14d
|
||||
- --storage.tsdb.retention.size=2GB
|
||||
volumes:
|
||||
- ./prometheus:/etc/prometheus:ro
|
||||
- prometheus_data:/prometheus
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: monitoring-grafana
|
||||
mem_limit: 256m
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001/}
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
ports:
|
||||
- "127.0.0.1:3001:3000"
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
# ------------------------------------------------------------------ logs --
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
container_name: monitoring-loki
|
||||
mem_limit: 384m
|
||||
command: -config.file=/etc/loki/loki-config.yml
|
||||
volumes:
|
||||
- ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro
|
||||
- loki_data:/loki
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
promtail:
|
||||
image: grafana/promtail:latest
|
||||
container_name: monitoring-promtail
|
||||
mem_limit: 64m
|
||||
command: -config.file=/etc/promtail/promtail-config.yml
|
||||
volumes:
|
||||
- ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------- exporters --
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
container_name: monitoring-node-exporter
|
||||
mem_limit: 32m
|
||||
command:
|
||||
- --path.rootfs=/host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
pid: host
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
postgres-exporter:
|
||||
image: prometheuscommunity/postgres-postgres-exporter:latest
|
||||
container_name: monitoring-postgres-exporter
|
||||
mem_limit: 48m
|
||||
environment:
|
||||
# App database on the main stack's postgres (reaches it via erp_shared).
|
||||
DATA_SOURCE_NAME: postgres://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/procurement?sslmode=disable
|
||||
networks:
|
||||
- monitoring
|
||||
- shared
|
||||
restart: unless-stopped
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
container_name: monitoring-cadvisor
|
||||
mem_limit: 128m
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
networks:
|
||||
- monitoring
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
driver: bridge
|
||||
# Reaches the ERP containers (api:8000/metrics, postgres:5432) — the main
|
||||
# compose declares the same fixed-name network as "shared".
|
||||
shared:
|
||||
name: erp_shared
|
||||
external: true
|
||||
# Created by the main compose; required only for the public GlitchTip route.
|
||||
traefik:
|
||||
name: traefik_network
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
glitchtip_postgres_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"annotations": { "list": [] },
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (status) (rate(http_requests_total{job=\"erp-api\"}[5m]))",
|
||||
"legendFormat": "HTTP {{status}}"
|
||||
}
|
||||
],
|
||||
"title": "API — request rate by status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 10, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 1 },
|
||||
{ "color": "red", "value": 2 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"id": 2,
|
||||
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket{job=\"erp-api\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
],
|
||||
"title": "API — latency (p50 / p95)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "fillOpacity": 15, "lineWidth": 2, "showPoints": "never" },
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"id": 3,
|
||||
"options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (name) (container_memory_working_set_bytes{job=\"cadvisor\", name!=\"\", image!=\"\"})",
|
||||
"legendFormat": "{{name}}"
|
||||
}
|
||||
],
|
||||
"title": "Containers — memory in use",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"id": 4,
|
||||
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum(pg_stat_activity_count)",
|
||||
"legendFormat": "postgres connections"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "host RAM available"
|
||||
}
|
||||
],
|
||||
"title": "Database connections / host RAM",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
|
||||
"id": 5,
|
||||
"options": {
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true,
|
||||
"showTime": true,
|
||||
"enableLogDetails": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"expr": "{compose_project=\"erp\"}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "All ERP container logs (Loki)",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["erp"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"hide": 2,
|
||||
"current": { "text": "Prometheus", "value": "Prometheus" }
|
||||
},
|
||||
{
|
||||
"name": "DS_LOKI",
|
||||
"label": "Loki",
|
||||
"type": "datasource",
|
||||
"query": "loki",
|
||||
"hide": 2,
|
||||
"current": { "text": "Loki", "value": "Loki" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "ERP — Stack overview",
|
||||
"uid": "erp-overview",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: erp-dashboards
|
||||
orgId: 1
|
||||
folder: ERP
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 60
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
@@ -0,0 +1,43 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
# Keep the log store bounded — this box shares 8 GB RAM / limited disk.
|
||||
retention_period: 14d
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burstsize_mb: 8
|
||||
max_query_series: 500
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
ruler:
|
||||
storage:
|
||||
type: local
|
||||
local:
|
||||
directory: /loki/rules
|
||||
@@ -0,0 +1,58 @@
|
||||
# Alert rules surfaced in Grafana's alerting UI (no Alertmanager needed at
|
||||
# this scale — Grafana can route them later via email/Telegram).
|
||||
groups:
|
||||
- name: erp-availability
|
||||
rules:
|
||||
- alert: ErpApiDown
|
||||
expr: up{job="erp-api"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ERP API is not scraping (down or /metrics failing)"
|
||||
|
||||
- alert: ErpApiHighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{job="erp-api", status=~"5.."}[5m]))
|
||||
/ clamp_min(sum(rate(http_requests_total{job="erp-api"}[5m])), 0.001) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "ERP API 5xx rate above 5% for 5 minutes"
|
||||
|
||||
- alert: ErpApiSlowRequests
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(http_request_duration_seconds_bucket{job="erp-api"}[5m])) by (le)) > 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "ERP API p95 latency above 2s for 10 minutes"
|
||||
|
||||
- name: host
|
||||
rules:
|
||||
- alert: HostDiskAlmostFull
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"}) < 0.1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Root filesystem below 10% free (deploy builds will fail)"
|
||||
|
||||
- alert: HostMemoryPressure
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Host RAM below 10% available (check monitoring stack limits)"
|
||||
|
||||
- alert: PostgresTooManyConnections
|
||||
expr: sum(pg_stat_activity_count) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PostgreSQL connection count above 80"
|
||||
@@ -0,0 +1,34 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
# The ERP API — /metrics is exposed by prometheus-fastapi-instrumentator
|
||||
# (request rate, latency histogram, error counts). Reached over the
|
||||
# erp_shared network the main compose also joins.
|
||||
- job_name: erp-api
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ["api:8000"]
|
||||
|
||||
# Host-level metrics (CPU, RAM, disk, network).
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
|
||||
# Per-container metrics (CPU, memory, restarts) for every container.
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ["cadvisor:8080"]
|
||||
|
||||
# App database metrics (connections, cache hit ratio, WAL, ...).
|
||||
- job_name: postgres
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,40 @@
|
||||
# Ships every container's JSON logs to Loki (docker service discovery),
|
||||
# tagging streams with the compose project / service / container name so
|
||||
# logs are filterable in Grafana, e.g. {compose_service="api"}.
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: docker
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 30s
|
||||
filters:
|
||||
- name: label
|
||||
values: ["com.docker.compose.service"]
|
||||
relabel_configs:
|
||||
- source_labels: ["__meta_docker_container_label_com_docker_compose_project"]
|
||||
target_label: compose_project
|
||||
- source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label: compose_service
|
||||
- source_labels: ["__meta_docker_container_name"]
|
||||
target_label: container
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
ts: time
|
||||
stream: stream
|
||||
log: log
|
||||
- labels:
|
||||
stream:
|
||||
- timestamp:
|
||||
source: ts
|
||||
format: RFC3339Nano
|
||||
- output:
|
||||
source: log
|
||||
Reference in New Issue
Block a user