feat(monitoring): observability stack + app instrumentation
- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate), /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture, X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT) - server-side analytics (OpenPanel Track API): login, quote lifecycle, receipts — fire-and-forget, disabled without env config - monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts), Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs, separate from app deploys; erp_shared network for scraping - frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when unset), funnel events (login, wizard steps, quote submitted) - compose: api healthcheck, observability env wiring, .env.example keys
This commit is contained in:
+79
-1
@@ -7,12 +7,16 @@
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from prometheus_fastapi_instrumentator import Instrumentator
|
||||
from sqlalchemy import text as sql_text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.auth import get_current_user
|
||||
@@ -25,13 +29,29 @@ from app.db.init_db import (
|
||||
ensure_quote_projects,
|
||||
ensure_v26,
|
||||
)
|
||||
from app.observability import RequestContextMiddleware, setup_logging
|
||||
from app.routers import ROUTERS
|
||||
from app.services.quote_service import auto_expire_quotes
|
||||
from app.utils.errors import AppError
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s")
|
||||
setup_logging(settings.LOG_FORMAT)
|
||||
logger = logging.getLogger("procurement")
|
||||
|
||||
if settings.SENTRY_DSN:
|
||||
# GlitchTip/Sentry exception capture — after setup_logging so the SDK's
|
||||
# own logs go through our formatter too. Disabled entirely without a DSN.
|
||||
import sentry_sdk
|
||||
from sentry_sdk.integrations.fastapi import FastApiIntegration
|
||||
|
||||
sentry_sdk.init(
|
||||
dsn=settings.SENTRY_DSN,
|
||||
environment=settings.ENVIRONMENT,
|
||||
traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
|
||||
integrations=[FastApiIntegration()],
|
||||
send_default_pii=False,
|
||||
)
|
||||
logger.info("Sentry/GlitchTip exception capture enabled")
|
||||
|
||||
API_V1_PREFIX = "/api/v1"
|
||||
|
||||
|
||||
@@ -107,6 +127,12 @@ app.add_middleware(
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
|
||||
# Prometheus /metrics (requests, latency, errors) — mounted at app level,
|
||||
# outside the authenticated /api/v1 prefix; scrape target for the
|
||||
# monitoring stack.
|
||||
Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -203,3 +229,55 @@ for router in ROUTERS:
|
||||
@app.get("/health", tags=["system"])
|
||||
async def health():
|
||||
return {"status": "ok", "service": "procurement-api"}
|
||||
|
||||
|
||||
async def _check_db() -> bool:
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
await session.execute(sql_text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("readiness: database check failed")
|
||||
return False
|
||||
|
||||
|
||||
async def _check_minio() -> bool:
|
||||
try:
|
||||
from app.services.storage import get_client
|
||||
|
||||
get_client().bucket_exists(settings.MINIO_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("readiness: MinIO check failed")
|
||||
return False
|
||||
|
||||
|
||||
async def _check_casdoor() -> bool:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
resp = await client.get(
|
||||
settings.CASDOOR_JWKS_URL or f"{settings.CASDOOR_ENDPOINT}/.well-known/jwks"
|
||||
)
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
logger.exception("readiness: Casdoor JWKS check failed")
|
||||
return False
|
||||
|
||||
|
||||
@app.get("/health/ready", tags=["system"])
|
||||
async def health_ready():
|
||||
"""Readiness: verifies the dependencies the API actually needs.
|
||||
|
||||
The deploy webhook polls this after a rebuild, so a deploy that breaks
|
||||
the DB/MinIO/Casdoor connectivity is visible immediately.
|
||||
"""
|
||||
checks = {
|
||||
"database": await _check_db(),
|
||||
"minio": await _check_minio(),
|
||||
"casdoor": await _check_casdoor(),
|
||||
}
|
||||
ok = all(checks.values())
|
||||
return JSONResponse(
|
||||
status_code=200 if ok else 503,
|
||||
content={"status": "ok" if ok else "degraded", "service": "procurement-api", "checks": checks},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user