Files
ERP/bot/app/handlers/quote.py
T
whqreza 1c853e50a9 feat(matching): Jev entity matching phases 2-5 (per-field fan-out, variants, candidate ambiguity buttons)
Phase 2 — jev.py client: POST /v1/systemone, PINNED model jev-1.13.0 (drift warning when the response model differs), one retry on 429/5xx, fail-soft (None on any error), usage/latency returned for observability. Config adds TYPESAFE_API_KEY / JEV_* (thresholds 0.85/0.60, 15 questions per field); no daily cap by owner decision. bot/.env is now mounted read-only into the container so the key never enters compose/git.

Phase 3 — matching.py: برون‌گروهی (one request per field, parallel), درون‌گروهی (one choice question per mention sharing the same full candidate list + «_none»), درون‌واریانسی (all STT spellings in one question). Routing: conf>=exact → bind; floor..exact → ambiguity; else/_none → new; missing/failed field → difflib fallback (bug fixed during testing: a failed request used to record 'new' instead of falling back). Resolver gains full cached lists for suppliers/materials (paged) plus brands (API+DB) and specs (EAV values + draft specs). NLU prompt now emits entity_mentions (6 fields) with the voice-variant grouping rule; sanitize_mentions guards the shape. handle_message runs matching only for register_quote/update_draft/add_material behind JEV_ENABLED; jev_map threads into quote/material resolve_payload (Jev hit skips difflib; exact writes ids; ambiguity skips difflib). Spec effect per owner: spelling normalization only (normalize_brand_spec).

Phase 4 — entity-match ambiguity sub-messages: create_ambiguities() builds deduped entries {kind: entity_match, field, variants, candidates, details with Persian percentages}; new keyboards.ambiguity_candidates_keyboard renders one button per candidate + «هیچ‌کدام — ایجاد جدید»; callbacks handle amb⛏️{prefix}:{amb_id}:{idx} (applies the candidate via matching.apply_choice then re-resolves/renders) and amb:new (rejected → auto-create path at commit). Every user choice lands in observability (ambiguity.user_choice with arg+candidates).

Phase 5 — voice variants: NLU groups different STT spellings of one entity into one variants[] (verified live: مصطفوی/مستفدی → one supplier mention); tools/test_matching.py live section (RUN_LIVE=1) exercises real DeepSeek + real Jev without touching Bale.

Tests: test_jev.py 5/5 (mock HTTP: no-key, happy path, 429-retry, 400, exception), test_matching.py 11/11 mock + 3/3 live (real NLU produced 7 mentions with correct fields; real Jev matched «50 کیلویی»→«۵۰ کیلویی» 0.98 and «کیسه» 1.0 and correctly returned new against the emptied catalog); observability regression 8/8.
2026-10-12 21:24:03 +03:30

529 lines
22 KiB
Python

"""Register-quote flow: entities → draft → resolve → review (+missing) → commit."""
import logging
from app import drafts as draft_store
from app import matching
from app import observability as obs
from app.api.client import ApiError
from app.drafts import (
KIND_QUOTE, build_quote_request, compute_quote_missing, merge_quote_patch,
new_quote_payload, quote_ready,
)
from app.keyboards import quote_review_keyboard
from app.resolve import safe_resolve
from app.texts import ERR_GENERIC, quote_committed_report, quote_review_text, esc
logger = logging.getLogger("bot.quote")
FIELD_FA = {"sup": "تأمین‌کننده", "prj": "پروژه"}
async def handle_entities(
ctx, chat_id, user, patch: dict, source: str | None,
incoming_msg_id: int | None = None,
target_draft = None,
trace_id: str | None = None,
jev_map: dict | None = None,
) -> None:
prev_review_msg_id = None
if target_draft is not None:
draft = target_draft
prev_review_msg_id = draft.review_message_id
draft.review_message_id = None
# Clean up old ambiguity sub-messages (Requirement 2.c)
old_ambs = (draft.payload or {}).get("ambiguities") or []
for a in old_ambs:
if a.get("message_id"):
try:
await ctx.delete(chat_id, a["message_id"])
except Exception:
pass
draft.payload["ambiguities"] = []
payload = draft.payload
merge_quote_patch(payload, patch)
draft.source_message_id = incoming_msg_id
else:
payload = new_quote_payload()
merge_quote_patch(payload, patch)
draft = await draft_store.create(
chat_id, KIND_QUOTE, payload, source=source, source_message_id=incoming_msg_id
)
await obs.link_draft(trace_id, draft.id, draft.kind)
await resolve_payload(ctx, payload, trace_id, jev_map)
if jev_map:
matching.normalize_brand_spec(payload, jev_map)
matching.create_ambiguities(payload, jev_map)
if prev_review_msg_id:
try:
await ctx.delete(chat_id, prev_review_msg_id)
except Exception as e:
logger.warning("Could not delete previous review message %s: %s", prev_review_msg_id, e)
await render(ctx, chat_id, draft, fresh=True, reply_to=incoming_msg_id, trace_id=trace_id)
# ---------------------------------------------------------------------------
# resolution
# ---------------------------------------------------------------------------
async def _log_res(trace_id, field: str, mention, res) -> None:
"""Observability: record one difflib resolution attempt (baseline for phase 6)."""
await obs.log_resolution(trace_id, field, mention, res.status,
[{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
jev_map: dict | None = None) -> None:
options = payload.setdefault("options", {})
if not payload.get("supplier_id") and payload.get("supplier_name"):
hit = matching.lookup(jev_map, payload["supplier_name"])
if hit is not None:
if hit["status"] == "exact":
matching.apply_exact(payload, hit)
options.pop("sup", None)
else:
res = await safe_resolve(ctx.resolver, "supplier", payload["supplier_name"])
await _log_res(trace_id, "supplier", payload["supplier_name"], res)
if res.resolved:
c = res.candidates[0]
payload["supplier_id"] = c.id
payload["supplier_name"] = c.name
extra = c.extra or {}
for k in ("location", "phone", "contact_person", "category_name"):
if extra.get(k):
payload[f"supplier_{k}"] = extra[k]
options.pop("sup", None)
elif res.candidates:
options["sup"] = [{"id": c.id, "name": c.name} for c in res.candidates]
else:
options.pop("sup", None)
resolved_projects = payload.setdefault("resolved_projects", {})
if not payload.get("project_id") and payload.get("project_name"):
hit = matching.lookup(jev_map, payload["project_name"])
if hit is not None:
if hit["status"] == "exact":
matching.apply_exact(payload, hit)
payload["project_name"] = hit["name"]
options.pop("prj", None)
else:
res = await safe_resolve(ctx.resolver, "project", payload["project_name"])
await _log_res(trace_id, "project", payload["project_name"], res)
if res.resolved:
c = res.candidates[0]
payload["project_id"] = c.id
payload["project_name"] = c.name
resolved_projects[c.name] = c.id
resolved_projects[payload["project_name"]] = c.id
if payload["project_id"] not in payload.get("project_ids", []):
payload.setdefault("project_ids", []).append(payload["project_id"])
options.pop("prj", None)
elif res.candidates:
options["prj"] = [{"id": c.id, "name": c.name} for c in res.candidates]
else:
options.pop("prj", None)
for p_name in payload.get("project_names") or []:
hit = matching.lookup(jev_map, p_name)
if hit is not None:
if hit["status"] == "exact":
matching.apply_exact(payload, hit)
continue
res = await safe_resolve(ctx.resolver, "project", p_name)
await _log_res(trace_id, "project", p_name, res)
if res.resolved:
c = res.candidates[0]
p_id = c.id
resolved_projects[c.name] = p_id
resolved_projects[p_name] = p_id
if p_id not in payload.get("project_ids", []):
payload.setdefault("project_ids", []).append(p_id)
if not payload.get("project_id"):
payload["project_id"] = p_id
for i, item in enumerate(payload.get("items") or [], start=1):
if not item.get("material_id") and item.get("material_name"):
hit = matching.lookup(jev_map, item["material_name"])
if hit is not None:
if hit["status"] == "exact":
matching.apply_exact(payload, hit)
options.pop(f"mat{i}", None)
else:
res = await safe_resolve(ctx.resolver, "material", item["material_name"])
await _log_res(trace_id, "material", item["material_name"], res)
if res.resolved:
_set_material(item, res.candidates[0].id, res.candidates[0].name, res.candidates[0].extra)
options.pop(f"mat{i}", None)
elif res.candidates:
options[f"mat{i}"] = [{"id": c.id, "name": c.name} for c in res.candidates]
else:
options.pop(f"mat{i}", None)
if not item.get("unit_id"):
await _resolve_unit(ctx, item, trace_id, jev_map)
def _set_material(item: dict, material_id: str, name: str, extra: dict | None = None) -> None:
item["material_id"] = material_id
item["material_row"] = {"id": material_id, "canonical_unit_id": (extra or {}).get("canonical_unit_id")}
async def _resolve_unit(ctx, item: dict, trace_id: str | None = None,
jev_map: dict | None = None) -> None:
if item.get("unit_id"):
return
if item.get("unit_name"):
hit = matching.lookup(jev_map, item["unit_name"])
if hit is not None:
if hit["status"] == "exact":
item["unit_id"] = hit["id"]
item["unit_explicit"] = True
item["unit_name"] = hit["name"]
return
else:
res = await safe_resolve(ctx.resolver, "unit", item["unit_name"])
await _log_res(trace_id, "unit", item["unit_name"], res)
if res.resolved:
item["unit_id"] = res.candidates[0].id
item["unit_explicit"] = True
item["unit_name"] = res.candidates[0].name
return
canonical = ((item.get("material_row") or {}).get("canonical_unit_id"))
if canonical:
item["unit_id"] = str(canonical)
item["unit_explicit"] = False
async def apply_pick(ctx, chat_id, user, draft, field: str, choice: dict,
trace_id: str | None = None) -> None:
payload = draft.payload
trace_id = trace_id or await obs.latest_trace_for_draft(draft.id)
options = payload.setdefault("options", {})
options.pop(field, None)
if field == "sup":
payload["supplier_id"] = choice["id"]
payload["supplier_name"] = choice["name"]
elif field == "prj":
payload["project_id"] = choice["id"]
payload["project_name"] = choice["name"]
elif field.startswith("mat"):
index = int(field[3:]) - 1
items = payload.get("items") or []
if 0 <= index < len(items):
items[index]["material_name"] = choice["name"]
_set_material(items[index], choice["id"], choice["name"])
await _resolve_unit(ctx, items[index], trace_id)
elif field.startswith("unit"):
index = int(field[4:]) - 1
items = payload.get("items") or []
if 0 <= index < len(items):
items[index]["unit_id"] = choice["id"]
items[index]["unit_explicit"] = True
items[index]["unit_name"] = choice["name"]
await resolve_payload(ctx, payload, trace_id)
await draft_store.save_payload(draft, missing={"fields": compute_quote_missing(payload)})
await render(ctx, chat_id, draft, fresh=False, trace_id=trace_id)
# ---------------------------------------------------------------------------
# rendering
# ---------------------------------------------------------------------------
def _pick_rows(payload: dict) -> list[list[str]]:
"""Field labels shown as pick-button rows (via keyboards.pick_buttons)."""
rows: list[list[str]] = []
options = payload.get("options") or {}
if options.get("sup"):
rows.append(["sup"] + [o["name"] for o in options["sup"]])
if options.get("prj"):
rows.append(["prj"] + [o["name"] for o in options["prj"]])
for i, item in enumerate(payload.get("items") or [], start=1):
if options.get(f"mat{i}"):
rows.append([f"mat{i}"] + [o["name"] for o in options[f"mat{i}"]])
if options.get(f"unit{i}"):
rows.append([f"unit{i}"] + [o["name"] for o in options[f"unit{i}"]])
return rows
def _keyboard(payload: dict, draft_id: str, wizard_url: str | None = None) -> dict:
from app.keyboards import pick_buttons
prefix = str(draft_id)[:8]
pick_rows: list[list[dict]] = []
for row in _pick_rows(payload):
field, names = row[0], row[1:]
pick_rows += pick_buttons(prefix, field, names[:6])
ready_items = sum(
1 for it in payload.get("items") or []
if it.get("material_id") and it.get("quantity") and it.get("unit_id")
)
has_partial = 0 < ready_items < len(payload.get("items") or [])
return quote_review_keyboard(prefix, pick_rows, has_partial_option=has_partial, wizard_url=wizard_url)
async def render(ctx, chat_id, draft, fresh: bool = True, note: str | None = None,
reply_to: int | None = None, trace_id: str | None = None) -> None:
from app.config import settings
from app.security import create_draft_token
payload = draft.payload
missing = compute_quote_missing(payload)
token = create_draft_token(
draft.id,
chat_id=chat_id,
secret=settings.DRAFT_TOKEN_SECRET,
expires_in=settings.DRAFT_TOKEN_EXPIRE_SECONDS,
)
wizard_url = f"{settings.FRONTEND_URL.rstrip('/')}/review?token={token}"
text = quote_review_text(payload, missing, status_note=note)
text += f"\n\n🖥 مشاهده و تکمیل در ویزارد:\n{wizard_url}"
keyboard = _keyboard(payload, str(draft.id), wizard_url=wizard_url)
if fresh:
message = await ctx.send(chat_id, text, reply_markup=keyboard, reply_to=reply_to)
new_review_id = (message or {}).get("message_id")
if new_review_id:
draft.review_message_id = new_review_id
await draft_store.save_payload(
draft,
missing={"fields": missing},
review_message_id=new_review_id,
source_message_id=reply_to or draft.source_message_id,
mark_bale_synced=True,
)
else:
await draft_store.save_payload(
draft,
missing={"fields": missing},
source_message_id=reply_to or draft.source_message_id,
mark_bale_synced=True,
)
# Send pending ambiguity sub-messages as replies to the main review card
await send_pending_ambiguities(ctx, chat_id, draft)
else:
if not draft.review_message_id:
return
success = await ctx.edit(chat_id, draft.review_message_id, text, reply_markup=keyboard)
if success:
await draft_store.save_payload(draft, missing={"fields": missing}, mark_bale_synced=True)
else:
logger.warning("Could not edit review message %s for draft %s", draft.review_message_id, draft.id)
await draft_store.set_bale_updated_at(draft.id)
await obs.event(trace_id or await obs.latest_trace_for_draft(draft.id), "draft.render",
payload={"fresh": fresh, "missing": [m.get("key") for m in missing if not m.get("ok")]})
async def send_pending_ambiguities(ctx, chat_id: int, draft) -> None:
from app.keyboards import ambiguity_keyboard
from app.texts import ambiguity_message_text
if not draft.review_message_id:
return
prefix = str(draft.id)[:8]
payload = draft.payload or {}
ambiguities = payload.get("ambiguities") or []
updated = False
trace_id = await obs.latest_trace_for_draft(draft.id)
for amb in ambiguities:
if amb.get("status") == "pending" and not amb.get("message_id"):
msg_text = ambiguity_message_text(amb.get("title", ""), amb.get("details", ""))
kb = ambiguity_keyboard(prefix, amb.get("id", ""))
sent = await ctx.send(
chat_id,
msg_text,
reply_markup=kb,
reply_to=draft.review_message_id,
)
if sent and sent.get("message_id"):
amb["message_id"] = sent["message_id"]
updated = True
await obs.event(trace_id, "ambiguity.sent", payload={
"amb_id": amb.get("id"), "title": amb.get("title"),
"kind": amb.get("kind"), "message_id": sent["message_id"],
})
if updated:
await draft_store.save_payload(draft, mark_bale_synced=True)
# ---------------------------------------------------------------------------
# commit
# ---------------------------------------------------------------------------
async def commit(ctx, chat_id, user, draft, only_resolved: bool = False,
trace_id: str | None = None) -> None:
payload = draft.payload
trace_id = trace_id or await obs.latest_trace_for_draft(draft.id)
await resolve_payload(ctx, payload, trace_id)
newly_created: dict = {}
# 1. Sequential pre-creation for new supplier if user wants to commit
if not payload.get("supplier_id") and payload.get("supplier_name"):
cat_id = payload.get("supplier_category_id")
if not cat_id:
cats = await ctx.resolver.categories()
if cats:
cat_id = cats[0].id
if cat_id:
try:
sup_body = {
"name": payload["supplier_name"],
"category_id": cat_id,
}
if payload.get("supplier_phone"):
sup_body["phone"] = payload["supplier_phone"]
created_sup = await ctx.api.create_supplier(sup_body, actor_id=user.actor_id)
payload["supplier_id"] = created_sup["id"]
newly_created["supplier"] = payload["supplier_name"]
ctx.resolver.invalidate("suppliers")
except Exception as e:
logger.warning("Could not auto-create supplier: %s", e)
# 2. Sequential pre-creation for new materials
created_mats = []
for i, item in enumerate(payload.get("items") or [], start=1):
if not item.get("material_id") and item.get("material_name") and item.get("unit_id"):
try:
created_mat = await ctx.api.create_material({
"name": item["material_name"],
"canonical_unit_id": int(item["unit_id"]),
}, actor_id=user.actor_id)
_set_material(item, created_mat["id"], created_mat.get("name") or item["material_name"])
created_mats.append(item["material_name"])
ctx.resolver.invalidate("materials")
except Exception as e:
logger.warning("Could not auto-create material: %s", e)
if created_mats:
newly_created["materials"] = created_mats
# 3. Sequential pre-creation for new projects
created_prjs = []
all_p_names = []
if payload.get("project_names"):
for p in payload["project_names"]:
p_str = str(p).strip()
if p_str and p_str not in all_p_names:
all_p_names.append(p_str)
elif payload.get("project_name"):
p_str = str(payload["project_name"]).strip()
if p_str and p_str not in all_p_names:
all_p_names.append(p_str)
for it in payload.get("items") or []:
it_p = it.get("project_name")
if it_p:
it_p_str = str(it_p).strip()
if it_p_str and it_p_str not in all_p_names:
all_p_names.append(it_p_str)
resolved_map = payload.setdefault("resolved_projects", {})
for p_name in all_p_names:
p_id = resolved_map.get(p_name)
if not p_id:
res = await safe_resolve(ctx.resolver, "project", p_name)
if res.resolved:
p_id = res.candidates[0].id
resolved_map[p_name] = p_id
else:
try:
created_prj = await ctx.api.create_project({"name": p_name}, actor_id=user.actor_id)
p_id = created_prj.get("id")
if p_id:
resolved_map[p_name] = p_id
created_prjs.append(p_name)
ctx.resolver.invalidate("projects")
except Exception as e:
logger.warning("Could not auto-create project %s: %s", p_name, e)
if p_id:
if p_id not in payload.get("project_ids", []):
payload.setdefault("project_ids", []).append(p_id)
if not payload.get("project_id"):
payload["project_id"] = p_id
if created_prjs:
newly_created["projects"] = created_prjs
ready, blockers = quote_ready(payload)
body, problems = build_quote_request(payload, only_resolved_items=only_resolved)
if body is None:
lines = ["ثبت ممکن نیست؛ این‌ها را اول مشخص کن:"] + [f"❌ {esc(p)}" for p in problems]
await ctx.send(chat_id, "\n".join(lines))
return
if not ready and not only_resolved:
lines = ["برای ثبت کامل، اول این‌ها را مشخص کن:"] + [f"❌ {esc(b)}" for b in blockers]
lines += ["", "یا دکمه «ثبت فقط اقلام مشخص» را بزن."]
await ctx.send(chat_id, "\n".join(lines))
return
try:
created = await ctx.api.create_quote(body, actor_id=user.actor_id)
except ApiError as exc:
await ctx.send(chat_id, ERR_GENERIC.format(detail=esc(exc.message)))
return
except Exception:
logger.exception("create_quote failed")
await ctx.send(chat_id, ERR_GENERIC.format(detail="خطای شبکه"))
return
await draft_store.set_status(draft.id, "committed")
ctx.pending.pop(chat_id, None)
ctx.resolver.invalidate()
quote_id = (created or {}).get("id") or ""
await obs.event(trace_id, "draft.commit", payload={
"quote_id": quote_id,
"items": len(body.get("line_items") or []),
"newly_created": newly_created,
"only_resolved": only_resolved,
})
await obs.set_trace_status(trace_id, "committed", quote_id)
# If a proforma file was attached, persist it to the created ERP quote
proforma_info = payload.get("proforma")
if quote_id and proforma_info and proforma_info.get("saved_path"):
try:
from pathlib import Path
p_path = Path(proforma_info["saved_path"])
if p_path.exists():
file_bytes = p_path.read_bytes()
await ctx.api.upload_quote_file(
quote_id,
"proforma",
proforma_info.get("file_name") or "proforma.pdf",
file_bytes,
mime_type=proforma_info.get("mime_type") or "application/pdf",
actor_id=user.actor_id,
)
logger.info("Successfully uploaded proforma to quote %s", quote_id)
except Exception as e:
logger.warning("Could not upload draft proforma to quote %s: %s", quote_id, e)
if draft.review_message_id:
try:
msg = f"✅ این پیشنهاد در سیستم با موفقیت ثبت نهایی شد.\nشناسه پیشنهاد: {quote_id}"
await ctx.edit(chat_id, draft.review_message_id, msg, reply_markup=None)
await draft_store.set_bale_updated_at(draft.id)
except Exception:
pass
remaining = [
m["label"] for m in compute_quote_missing(payload)
if not m["blocking"] and not m["ok"]
]
dropped = len(payload.get("items") or []) - len(body.get("line_items") or [])
if dropped > 0:
remaining.append(f"{dropped} قلم ناشناس حذف شد")
await ctx.send(chat_id, quote_committed_report(created, payload, remaining, newly_created=newly_created))