feat(monitoring): observability stack + app instrumentation

- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate),
  /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture,
  X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT)
- server-side analytics (OpenPanel Track API): login, quote lifecycle,
  receipts — fire-and-forget, disabled without env config
- monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts),
  Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres
  exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs,
  separate from app deploys; erp_shared network for scraping
- frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when
  unset), funnel events (login, wizard steps, quote submitted)
- compose: api healthcheck, observability env wiring, .env.example keys
This commit is contained in:
MMD
2026-09-18 22:18:02 +03:30
parent f7b1f7f751
commit e17d1f0883
26 changed files with 1106 additions and 3 deletions
+79 -1
View File
@@ -7,12 +7,16 @@
"""
import asyncio
import logging
import uuid
from contextlib import asynccontextmanager
import httpx
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from prometheus_fastapi_instrumentator import Instrumentator
from sqlalchemy import text as sql_text
from sqlalchemy.exc import IntegrityError
from app.auth import get_current_user
@@ -25,13 +29,29 @@ from app.db.init_db import (
ensure_quote_projects,
ensure_v26,
)
from app.observability import RequestContextMiddleware, setup_logging
from app.routers import ROUTERS
from app.services.quote_service import auto_expire_quotes
from app.utils.errors import AppError
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s")
setup_logging(settings.LOG_FORMAT)
logger = logging.getLogger("procurement")
if settings.SENTRY_DSN:
# GlitchTip/Sentry exception capture — after setup_logging so the SDK's
# own logs go through our formatter too. Disabled entirely without a DSN.
import sentry_sdk
from sentry_sdk.integrations.fastapi import FastApiIntegration
sentry_sdk.init(
dsn=settings.SENTRY_DSN,
environment=settings.ENVIRONMENT,
traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
integrations=[FastApiIntegration()],
send_default_pii=False,
)
logger.info("Sentry/GlitchTip exception capture enabled")
API_V1_PREFIX = "/api/v1"
@@ -107,6 +127,12 @@ app.add_middleware(
allow_methods=["*"],
allow_headers=["*"],
)
app.add_middleware(RequestContextMiddleware)
# Prometheus /metrics (requests, latency, errors) — mounted at app level,
# outside the authenticated /api/v1 prefix; scrape target for the
# monitoring stack.
Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
# ---------------------------------------------------------------------------
@@ -203,3 +229,55 @@ for router in ROUTERS:
@app.get("/health", tags=["system"])
async def health():
return {"status": "ok", "service": "procurement-api"}
async def _check_db() -> bool:
try:
async with AsyncSessionLocal() as session:
await session.execute(sql_text("SELECT 1"))
return True
except Exception:
logger.exception("readiness: database check failed")
return False
async def _check_minio() -> bool:
try:
from app.services.storage import get_client
get_client().bucket_exists(settings.MINIO_BUCKET)
return True
except Exception:
logger.exception("readiness: MinIO check failed")
return False
async def _check_casdoor() -> bool:
try:
async with httpx.AsyncClient(timeout=5) as client:
resp = await client.get(
settings.CASDOOR_JWKS_URL or f"{settings.CASDOOR_ENDPOINT}/.well-known/jwks"
)
return resp.status_code == 200
except Exception:
logger.exception("readiness: Casdoor JWKS check failed")
return False
@app.get("/health/ready", tags=["system"])
async def health_ready():
"""Readiness: verifies the dependencies the API actually needs.
The deploy webhook polls this after a rebuild, so a deploy that breaks
the DB/MinIO/Casdoor connectivity is visible immediately.
"""
checks = {
"database": await _check_db(),
"minio": await _check_minio(),
"casdoor": await _check_casdoor(),
}
ok = all(checks.values())
return JSONResponse(
status_code=200 if ok else 503,
content={"status": "ok" if ok else "degraded", "service": "procurement-api", "checks": checks},
)