Files
ERP/bot/app/models.py
T
whqreza d58b619107 feat(observability): phase-1 silent pipeline recorder + phase-0 Jev validation
Phase 0 — Jev validated with the real key (5 Persian scenarios, api.typesafe.ai): stale project name «منشیانه»->«منشانه» conf 0.73 (ambiguity band), STT spelling variants «مصطفوی/مستفدی/مسطفی»->«بازرگانی مصطفوی» conf 0.93, exact «دفتر» 0.88, proper _none rejection for «کاغذ A4» 0.92, and two mentions in ONE request (0.77/0.64). Exact model version to pin: jev-1.13.0. Latency 0.7-1.1s per field request; ~450-650 in / 64-133 out tokens.

Phase 1 — Observability (DB-only, zero extra LLM tokens, nothing is ever deleted):

- new tables bot_traces + bot_trace_events (idempotent DDL in db_init.py; ORM in models.py)

- new app/observability.py: _jsonable (UUID/datetime/Decimal/set/enum/dataclass-safe), trace_start, event, log_resolution, link_draft, latest_trace_for_draft, set_trace_status, span() context manager, save_voice (Path.mkdir parents+exist_ok, day folders)

- capture points wired: input.text; input.voice (+voice persisted under OBS_VOICE_DIR); stt.run + per-provider stt.result (success AND failure, latency, model); nlu.request + nlu.result with prompt_hash/prompt_version; entities.extracted; routing.decision (difflib baseline for the phase-6 A/B); ambiguity.sent; ambiguity.user_choice (buttons + free-text reply path); user_choice.pick; draft.render; draft.edit (wizard-side edits detected by the sync loop); draft.commit (quote/supplier/material/project + trace status/quote id); draft.cancel

- compose: persistent ./bot/data:/srv/bot/data volume; bot/.gitignore: data/ (runtime data never committed, never deleted)

- tests: bot/tools/test_observability.py -> 8/8 pass (jsonable cases, trace lifecycle, JSONB round-trip, draft linking, status update, save_voice day-folder mkdir, span task). Verified live: voice file lands on the host volume and is gitignored; rows + files persist across bot restart.
2026-10-12 21:04:50 +03:30

130 lines
5.5 KiB
Python

"""Bot-owned tables (bot_* prefix, idempotent DDL in db_init.py).
These are separate from the ERP schema on purpose: the bot is an autonomous
container and owns its own state (links, drafts, message audit log).
"""
from datetime import datetime
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, String, Text, func
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
def _uuid_pk() -> Mapped[str]:
return mapped_column(UUID(as_uuid=False), primary_key=True, server_default=func.gen_random_uuid())
def _timestamptz(**kwargs) -> Mapped[datetime]:
return mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False, **kwargs)
class BotLink(Base):
"""Bale chat ↔ ERP user mapping (established via Casdoor login)."""
__tablename__ = "bot_links"
id: Mapped[str] = _uuid_pk()
chat_id: Mapped[int] = mapped_column(BigInteger, unique=True, nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(UUID(as_uuid=False)) # users.id
casdoor_username: Mapped[str | None] = mapped_column(Text)
display_name: Mapped[str | None] = mapped_column(Text)
role: Mapped[str | None] = mapped_column(String(32))
created_at: Mapped[datetime] = _timestamptz()
updated_at: Mapped[datetime] = _timestamptz(onupdate=func.now())
class BotDraft(Base):
"""A prepared-but-not-committed ERP action (quote / supplier / material)."""
__tablename__ = "bot_drafts"
__table_args__ = (
Index("ix_bot_drafts_chat_status", "chat_id", "status"),
Index("ix_bot_drafts_review_msg", "review_message_id"),
Index("ix_bot_drafts_source_msg", "source_message_id"),
)
id: Mapped[str] = _uuid_pk()
chat_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
kind: Mapped[str] = mapped_column(String(16), nullable=False) # quote|supplier|material
payload: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
missing: Mapped[dict | None] = mapped_column(JSONB)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="active") # active|committed|cancelled|expired|failed
source: Mapped[str | None] = mapped_column(String(16)) # text|voice
source_message_id: Mapped[int | None] = mapped_column(BigInteger)
review_message_id: Mapped[int | None] = mapped_column(BigInteger)
last_bale_update_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = _timestamptz()
updated_at: Mapped[datetime] = _timestamptz(onupdate=func.now())
class BotMessage(Base):
"""Audit log of every inbound/outbound message (who asked for what)."""
__tablename__ = "bot_messages"
id: Mapped[str] = _uuid_pk()
chat_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
direction: Mapped[str] = mapped_column(String(4), nullable=False) # in|out
kind: Mapped[str] = mapped_column(String(16), nullable=False, default="text") # text|voice|callback|error
body: Mapped[str | None] = mapped_column(Text)
payload: Mapped[dict | None] = mapped_column(JSONB)
created_at: Mapped[datetime] = _timestamptz()
class BotMeta(Base):
"""Tiny KV store (e.g. last processed Bale update_id across restarts)."""
__tablename__ = "bot_meta"
key: Mapped[str] = mapped_column(String(64), primary_key=True)
value: Mapped[str | None] = mapped_column(Text)
updated_at: Mapped[datetime] = _timestamptz(onupdate=func.now())
class BotTrace(Base):
"""Observability: one row per incoming interaction (message) — the chain root.
Events of the whole pipeline hang off it (bot_trace_events). Linked to the
draft it produced so later button clicks can be attributed to the same chain.
"""
__tablename__ = "bot_traces"
id: Mapped[str] = _uuid_pk()
chat_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
message_id: Mapped[int | None] = mapped_column(BigInteger)
kind: Mapped[str] = mapped_column(String(8), nullable=False) # text | voice
source: Mapped[str | None] = mapped_column(String(16)) # bale | wizard | api
draft_id: Mapped[str | None] = mapped_column(UUID(as_uuid=False))
draft_kind: Mapped[str | None] = mapped_column(String(16))
committed_quote_id: Mapped[str | None] = mapped_column(UUID(as_uuid=False))
status: Mapped[str] = mapped_column(String(16), nullable=False, default="open") # open|committed|cancelled
created_at: Mapped[datetime] = _timestamptz()
updated_at: Mapped[datetime] = _timestamptz(onupdate=func.now())
class BotTraceEvent(Base):
"""Observability: append-only pipeline event log (payload schemas by `stage`).
Written ONLY through app.observability (fire-and-forget, _jsonable-safe).
Nothing is ever deleted from here by the bot.
"""
__tablename__ = "bot_trace_events"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
trace_id: Mapped[str] = mapped_column(
UUID(as_uuid=False), ForeignKey("bot_traces.id", ondelete="CASCADE"), nullable=False
)
stage: Mapped[str] = mapped_column(String(32), nullable=False)
ok: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
latency_ms: Mapped[int | None] = mapped_column(Integer)
model: Mapped[str | None] = mapped_column(String(64))
field: Mapped[str | None] = mapped_column(String(32))
payload: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
created_at: Mapped[datetime] = _timestamptz()