feat(monitoring): observability stack + app instrumentation

- /health/ready (DB+MinIO+Casdoor checks, webhook post-deploy gate),
  /metrics via prometheus-fastapi-instrumentator, Sentry/GlitchTip capture,
  X-Request-ID middleware + JSON access logging (PROCUREMENT_LOG_FORMAT)
- server-side analytics (OpenPanel Track API): login, quote lifecycle,
  receipts — fire-and-forget, disabled without env config
- monitoring/ compose project: GlitchTip(+pg), Prometheus(+alerts),
  Grafana(provisioned ERP dashboard), Loki+Promtail, node/postgres
  exporters, cAdvisor — mem-limited (~2.2GB), loopback admin UIs,
  separate from app deploys; erp_shared network for scraping
- frontend: @openpanel/web + @sentry/react (build-arg gated, no-op when
  unset), funnel events (login, wizard steps, quote submitted)
- compose: api healthcheck, observability env wiring, .env.example keys

backend/app/config.py

@@ -78,6 +78,8 @@ class Settings(BaseSettings):
     INVENTORY_DEFAULT_PROJECT_ID: str = ""
 
     # ---- Observability (Phase 2) -------------------------------------------
+    # "text" (human console) | "json" (JSON lines for Loki/Promtail).
+    LOG_FORMAT: str = "text"
     # Sentry/GlitchTip DSN — backend exception capture (empty = disabled).
     SENTRY_DSN: str = ""
     SENTRY_TRACES_SAMPLE_RATE: float = 0.0

backend/app/main.py

@@ -7,12 +7,16 @@
 """
 import asyncio
 import logging
+import uuid
 from contextlib import asynccontextmanager
 
+import httpx
 from fastapi import Depends, FastAPI, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
+from prometheus_fastapi_instrumentator import Instrumentator
+from sqlalchemy import text as sql_text
 from sqlalchemy.exc import IntegrityError
 
 from app.auth import get_current_user
@@ -25,13 +29,29 @@ from app.db.init_db import (
     ensure_quote_projects,
     ensure_v26,
 )
+from app.observability import RequestContextMiddleware, setup_logging
 from app.routers import ROUTERS
 from app.services.quote_service import auto_expire_quotes
 from app.utils.errors import AppError
 
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s")
+setup_logging(settings.LOG_FORMAT)
 logger = logging.getLogger("procurement")
 
+if settings.SENTRY_DSN:
+    # GlitchTip/Sentry exception capture — after setup_logging so the SDK's
+    # own logs go through our formatter too. Disabled entirely without a DSN.
+    import sentry_sdk
+    from sentry_sdk.integrations.fastapi import FastApiIntegration
+
+    sentry_sdk.init(
+        dsn=settings.SENTRY_DSN,
+        environment=settings.ENVIRONMENT,
+        traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
+        integrations=[FastApiIntegration()],
+        send_default_pii=False,
+    )
+    logger.info("Sentry/GlitchTip exception capture enabled")
+
 API_V1_PREFIX = "/api/v1"
 
 
@@ -107,6 +127,12 @@ app.add_middleware(
     allow_methods=["*"],
     allow_headers=["*"],
 )
+app.add_middleware(RequestContextMiddleware)
+
+# Prometheus /metrics (requests, latency, errors) — mounted at app level,
+# outside the authenticated /api/v1 prefix; scrape target for the
+# monitoring stack.
+Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
 
 
 # ---------------------------------------------------------------------------
@@ -203,3 +229,55 @@ for router in ROUTERS:
 @app.get("/health", tags=["system"])
 async def health():
     return {"status": "ok", "service": "procurement-api"}
+
+
+async def _check_db() -> bool:
+    try:
+        async with AsyncSessionLocal() as session:
+            await session.execute(sql_text("SELECT 1"))
+        return True
+    except Exception:
+        logger.exception("readiness: database check failed")
+        return False
+
+
+async def _check_minio() -> bool:
+    try:
+        from app.services.storage import get_client
+
+        get_client().bucket_exists(settings.MINIO_BUCKET)
+        return True
+    except Exception:
+        logger.exception("readiness: MinIO check failed")
+        return False
+
+
+async def _check_casdoor() -> bool:
+    try:
+        async with httpx.AsyncClient(timeout=5) as client:
+            resp = await client.get(
+                settings.CASDOOR_JWKS_URL or f"{settings.CASDOOR_ENDPOINT}/.well-known/jwks"
+            )
+            return resp.status_code == 200
+    except Exception:
+        logger.exception("readiness: Casdoor JWKS check failed")
+        return False
+
+
+@app.get("/health/ready", tags=["system"])
+async def health_ready():
+    """Readiness: verifies the dependencies the API actually needs.
+
+    The deploy webhook polls this after a rebuild, so a deploy that breaks
+    the DB/MinIO/Casdoor connectivity is visible immediately.
+    """
+    checks = {
+        "database": await _check_db(),
+        "minio": await _check_minio(),
+        "casdoor": await _check_casdoor(),
+    }
+    ok = all(checks.values())
+    return JSONResponse(
+        status_code=200 if ok else 503,
+        content={"status": "ok" if ok else "degraded", "service": "procurement-api", "checks": checks},
+    )

backend/app/observability.py

@@ -0,0 +1,86 @@
+"""Observability plumbing — logging format, request IDs, request log.
+
+- ``setup_logging()``: human-readable text by default, JSON lines when
+  ``PROCUREMENT_LOG_FORMAT=json`` (what Promtail/Loki prefers).
+- ``request_context`` middleware: assigns an X-Request-ID to every request
+  (honours one sent by the edge), echoes it in the response, and logs one
+  structured line per request (method, path, status, duration, request id)
+  — this is what makes access logs useful in Loki.
+"""
+import contextvars
+import json
+import logging
+import time
+import uuid
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+request_id_var: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="-")
+
+_SKIP_ACCESS_LOG = {"/health", "/health/ready", "/metrics"}
+
+
+class JsonFormatter(logging.Formatter):
+    """Minimal JSON-lines formatter for Loki/Promtail ingestion."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload = {
+            "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "request_id": request_id_var.get(),
+        }
+        if record.exc_info:
+            payload["exception"] = self.formatException(record.exc_info)
+        for key in ("http_method", "http_path", "http_status", "duration_ms"):
+            value = getattr(record, key, None)
+            if value is not None:
+                payload[key] = value
+        return json.dumps(payload, ensure_ascii=False)
+
+
+def setup_logging(log_format: str = "text", level: int = logging.INFO) -> None:
+    formatter = JsonFormatter() if log_format.lower() == "json" else logging.Formatter(
+        "%(asctime)s %(levelname)s %(name)s — %(message)s"
+    )
+    root = logging.getLogger()
+    for handler in root.handlers[:]:
+        root.removeHandler(handler)
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    root.addHandler(handler)
+    root.setLevel(level)
+
+
+class RequestContextMiddleware(BaseHTTPMiddleware):
+    """X-Request-ID + one access-log line per request."""
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        rid = request.headers.get("x-request-id") or uuid.uuid4().hex
+        token = request_id_var.set(rid)
+        started = time.perf_counter()
+        try:
+            response = await call_next(request)
+        except Exception:
+            duration_ms = round((time.perf_counter() - started) * 1000, 1)
+            logging.getLogger("procurement.access").exception(
+                "unhandled exception",
+                extra={"http_method": request.method, "http_path": request.url.path,
+                       "http_status": 500, "duration_ms": duration_ms},
+            )
+            request_id_var.reset(token)
+            raise
+        duration_ms = round((time.perf_counter() - started) * 1000, 1)
+        response.headers["X-Request-ID"] = rid
+        request_id_var.reset(token)
+        if request.url.path not in _SKIP_ACCESS_LOG:
+            logging.getLogger("procurement.access").info(
+                "%s %s -> %s",
+                request.method, request.url.path, response.status_code,
+                extra={"http_method": request.method, "http_path": request.url.path,
+                       "http_status": response.status_code, "duration_ms": duration_ms},
+            )
+        return response

backend/app/routers/auth.py

@@ -21,6 +21,7 @@ from app.auth import _verify_token, context_from_claims
 from app.config import settings
 from app.db.engine import get_db
 from app.routers.users import _user_projects, user_to_dict
+from app.services import analytics
 from app.utils.errors import ServiceUnavailableError, UnauthorizedError
 
 logger = logging.getLogger("procurement.auth")
@@ -75,6 +76,15 @@ async def login(payload: LoginRequest, db: AsyncSession = Depends(get_db)):
     claims = await _verify_token(access_token)
     ctx = await context_from_claims(claims, db)
     projects = await _user_projects(db, ctx.user)
+    await analytics.track(
+        "login_success",
+        {"username": payload.username, "role": ctx.role},
+        profile_id=ctx.user.casdoor_id,
+    )
+    await analytics.identify(
+        ctx.user.casdoor_id,
+        {"name": ctx.display_name, "role": ctx.role, "username": payload.username},
+    )
     return {
         "data": {
             "access_token": access_token,

backend/app/routers/pm.py

@@ -50,6 +50,7 @@ from app.models.procurement import (
     Supplier,
 )
 from app.schemas.procurement import PMReceiptCreate
+from app.services import analytics
 from app.services.inventory_service import canonical_specs_hash
 from app.utils.errors import (
     ForbiddenError,
@@ -429,6 +430,16 @@ async def pm_confirm_receipt(
             quote.arrived_date = payload.received_at
 
     item_state = arrival_state(li.quantity, already_received + payload.received_qty)
+    await analytics.track(
+        "receipt_confirmed",
+        {
+            "receipt_id": str(receipt.id),
+            "quote_id": str(quote.id),
+            "quantity": str(payload.received_qty),
+            "state": item_state,
+        },
+        profile_id=ctx.display_name,
+    )
     return {"data": {
         "receipt_id": str(receipt.id),
         "state": item_state,

backend/app/services/analytics.py

@@ -0,0 +1,76 @@
+"""Server-side product analytics — OpenPanel Track API.
+
+Fire-and-forget business events (quote lifecycle, receipts, logins) so the
+OpenPanel dashboards show what actually happens inside the ERP, not just
+page views. Never raises: analytics must not break a business flow.
+
+Configuration (all optional — disabled when any is missing):
+    PROCUREMENT_OPENPANEL_API_URL     e.g. http://openpanel-api:3333 (self-hosted)
+    PROCUREMENT_OPENPANEL_CLIENT_ID   project client id (write mode)
+    PROCUREMENT_OPENPANEL_CLIENT_SECRET  project client secret
+
+Contract (OpenPanel Track API): POST {API_URL}/track with headers
+``openpanel-client-id`` / ``openpanel-client-secret`` and a JSON body
+``{"type": "track"|"identify", "profileId": ..., "event"/"properties": ...}``.
+"""
+import logging
+from typing import Any
+
+import httpx
+
+from app.config import settings
+
+logger = logging.getLogger("procurement.analytics")
+
+_TIMEOUT = httpx.Timeout(3.0)
+
+
+def _configured() -> bool:
+    return bool(
+        settings.OPENPANEL_API_URL
+        and settings.OPENPANEL_CLIENT_ID
+        and settings.OPENPANEL_CLIENT_SECRET
+    )
+
+
+async def track(event: str, properties: dict[str, Any] | None = None, profile_id: str | None = None) -> None:
+    """Record a named event; swallows every error."""
+    if not _configured():
+        return
+    body = {
+        "type": "track",
+        "event": event,
+        "properties": properties or {},
+    }
+    if profile_id:
+        body["profileId"] = profile_id
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            await client.post(
+                f"{settings.OPENPANEL_API_URL.rstrip('/')}/track",
+                json=body,
+                headers={
+                    "openpanel-client-id": settings.OPENPANEL_CLIENT_ID,
+                    "openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET,
+                },
+            )
+    except Exception as exc:  # noqa: BLE001 — analytics must never raise
+        logger.debug("analytics track failed (%s): %s", event, exc)
+
+
+async def identify(profile_id: str, properties: dict[str, Any] | None = None) -> None:
+    """Create/update a user profile (e.g. on login)."""
+    if not _configured():
+        return
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            await client.post(
+                f"{settings.OPENPANEL_API_URL.rstrip('/')}/track",
+                json={"type": "identify", "profileId": profile_id, "properties": properties or {}},
+                headers={
+                    "openpanel-client-id": settings.OPENPANEL_CLIENT_ID,
+                    "openpanel-client-secret": settings.OPENPANEL_CLIENT_SECRET,
+                },
+            )
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("analytics identify failed: %s", exc)

backend/app/services/quote_service.py

@@ -4,6 +4,7 @@ Every mutation here runs in ONE ``async with db.begin()`` block; any failure
 rolls the whole operation back. All state changes log a quote_event in the
 same transaction.
 """
+import asyncio
 import logging
 from datetime import date, datetime, timezone
 from uuid import UUID
@@ -33,6 +34,7 @@ from app.schemas.procurement import (
     QuoteUpdate,
     RejectQuoteIn,
 )
+from app.services import analytics
 from app.services.material_service import ensure_brand_material_belong
 from app.utils.errors import ConflictError, NotFoundError, ValidationError
 
@@ -56,6 +58,19 @@ def utcnow() -> datetime:
 
 def _add_event(db: AsyncSession, quote_id: UUID, event_type: str, payload: dict | None = None, actor: str | None = None) -> None:
     db.add(QuoteEvent(quote_id=quote_id, event_type=event_type, payload=payload or {}, actor=actor))
+    # Product analytics (fire-and-forget, never blocks or raises): the whole
+    # quote lifecycle funnels through here, so one hook covers every
+    # status transition for the OpenPanel dashboards.
+    try:
+        asyncio.get_running_loop().create_task(
+            analytics.track(
+                f"quote_{event_type}",
+                {"quote_id": str(quote_id), "actor": actor, **(payload or {})},
+                profile_id=actor,
+            )
+        )
+    except RuntimeError:  # no running loop (defensive — callers are async)
+        pass
 
 
 async def get_quote_or_404(db: AsyncSession, quote_id: UUID) -> Quote: