feat(bot): add DeepSeek crisis fallback engine for Jev failure and remove difflib heuristics
- Add app/fallback_matcher.py for intelligent crisis entity matching with token and latency observability - Route Jev failure/timeouts in app/matching.py to DeepSeek fallback engine - Handle MatchingFailureError defensively in handlers/__init__.py with honest error message - Enforce strict 'new' entity status in handlers/quote.py and implement defensive get_or_create on commit - Clean up app/resolve.py to eliminate difflib and character ratio heuristics - Pass all smoke tests (59/59), fallback matching tests (15/15), and matching unit tests (13/13)
This commit is contained in:
@@ -229,16 +229,30 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
|
||||
await obs.event(trace_id, "entities.extracted",
|
||||
payload={"patch": patch, "intent": intent, "entity_mentions": mentions})
|
||||
|
||||
# Phase 3: Jev per-field matching — only when it can be applied (flag-gated;
|
||||
# a failure leaves jev_map empty and the difflib path runs unchanged).
|
||||
# Entity matching (Jev primary, DeepSeek crisis fallback)
|
||||
jev_map: dict = {}
|
||||
if settings.JEV_ENABLED and mentions and intent in ("register_quote", "update_draft", "add_material"):
|
||||
if mentions and intent in ("register_quote", "update_draft", "add_material"):
|
||||
try:
|
||||
from app import matching
|
||||
jev_map = await matching.resolve_mentions(ctx, patch, mentions, trace_id)
|
||||
jev_map = await matching.resolve_mentions(
|
||||
ctx, patch, mentions, trace_id=trace_id, raw_input=input_payload
|
||||
)
|
||||
except matching.MatchingFailureError:
|
||||
logger.error("entity matching failed completely (crisis fallback failed)")
|
||||
await ctx.send(
|
||||
chat_id,
|
||||
"⚠️ سامانه در پردازش هوشمند موجودیتها با اختلال مواجه شد. لطفاً دوباره پیام را ارسال کنید.",
|
||||
reply_to=message_id,
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
logger.exception("jev matching failed")
|
||||
jev_map = {}
|
||||
logger.exception("unexpected matching failure")
|
||||
await ctx.send(
|
||||
chat_id,
|
||||
"⚠️ سامانه در پردازش هوشمند موجودیتها با اختلال مواجه شد. لطفاً دوباره پیام را ارسال کنید.",
|
||||
reply_to=message_id,
|
||||
)
|
||||
return
|
||||
|
||||
if source == "voice":
|
||||
await ctx._log(chat_id, "in", "voice", consensus_text)
|
||||
|
||||
@@ -9,7 +9,7 @@ from app.drafts import (
|
||||
KIND_MATERIAL, merge_material_patch, material_missing, new_material_payload,
|
||||
)
|
||||
from app.keyboards import pick_buttons, simple_review_keyboard
|
||||
from app.resolve import safe_resolve
|
||||
from app.nlu import normalize_text
|
||||
from app.texts import ERR_GENERIC, esc, material_committed_report, material_review_text
|
||||
|
||||
logger = logging.getLogger("bot.material")
|
||||
@@ -45,16 +45,19 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
elif hit["status"] == "ambiguity" and hit.get("candidates"):
|
||||
options["unit"] = [{"id": c["id"], "name": c["name"]}
|
||||
for c in hit["candidates"][:6]]
|
||||
elif hit["status"] == "new":
|
||||
options.pop("unit", None)
|
||||
else:
|
||||
res = await safe_resolve(ctx.resolver, "unit", payload["unit_name"])
|
||||
await obs.log_resolution(trace_id, "unit", payload["unit_name"], res.status,
|
||||
[{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
|
||||
if res.resolved:
|
||||
payload["canonical_unit_id"] = res.candidates[0].id
|
||||
payload["unit_name"] = res.candidates[0].name
|
||||
try:
|
||||
units = await ctx.resolver.units()
|
||||
except Exception:
|
||||
units = []
|
||||
norm_u = normalize_text(payload["unit_name"])
|
||||
exact = [u for u in units if normalize_text(u.name) == norm_u or (u.extra or {}).get("code") == norm_u]
|
||||
if len(exact) == 1:
|
||||
payload["canonical_unit_id"] = exact[0].id
|
||||
payload["unit_name"] = exact[0].name
|
||||
options.pop("unit", None)
|
||||
elif res.candidates:
|
||||
options["unit"] = [{"id": c.id, "name": c.name} for c in res.candidates]
|
||||
else:
|
||||
options.pop("unit", None)
|
||||
|
||||
|
||||
+154
-70
@@ -10,6 +10,7 @@ from app.drafts import (
|
||||
new_quote_payload, quote_ready,
|
||||
)
|
||||
from app.keyboards import quote_review_keyboard
|
||||
from app.nlu import normalize_text
|
||||
from app.resolve import safe_resolve
|
||||
from app.texts import ERR_GENERIC, quote_committed_report, quote_review_text, esc
|
||||
|
||||
@@ -67,15 +68,9 @@ async def handle_entities(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolution
|
||||
# resolution (pure intelligence; zero local difflib heuristics)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _log_res(trace_id, field: str, mention, res) -> None:
|
||||
"""Observability: record one difflib resolution attempt (baseline for phase 6)."""
|
||||
await obs.log_resolution(trace_id, field, mention, res.status,
|
||||
[{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
|
||||
|
||||
|
||||
async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
jev_map: dict | None = None) -> None:
|
||||
options = payload.setdefault("options", {})
|
||||
@@ -86,9 +81,14 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
if hit["status"] == "exact":
|
||||
matching.apply_exact(payload, hit)
|
||||
options.pop("sup", None)
|
||||
elif hit["status"] == "ambiguity" and hit.get("candidates"):
|
||||
options["sup"] = [{"id": c["id"], "name": c["name"]} for c in hit["candidates"][:6]]
|
||||
elif hit["status"] == "new":
|
||||
options.pop("sup", None)
|
||||
if hit.get("inferred_category_slug"):
|
||||
payload["supplier_inferred_category_slug"] = hit["inferred_category_slug"]
|
||||
else:
|
||||
res = await safe_resolve(ctx.resolver, "supplier", payload["supplier_name"])
|
||||
await _log_res(trace_id, "supplier", payload["supplier_name"], res)
|
||||
if res.resolved:
|
||||
c = res.candidates[0]
|
||||
payload["supplier_id"] = c.id
|
||||
@@ -111,18 +111,25 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
if hit["status"] == "exact":
|
||||
matching.apply_exact(payload, hit)
|
||||
payload["project_name"] = hit["name"]
|
||||
resolved_projects[hit["name"]] = hit["id"]
|
||||
resolved_projects[payload["project_name"]] = hit["id"]
|
||||
if hit["id"] not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(hit["id"])
|
||||
options.pop("prj", None)
|
||||
elif hit["status"] == "ambiguity" and hit.get("candidates"):
|
||||
options["prj"] = [{"id": c["id"], "name": c["name"]} for c in hit["candidates"][:6]]
|
||||
elif hit["status"] == "new":
|
||||
options.pop("prj", None)
|
||||
else:
|
||||
res = await safe_resolve(ctx.resolver, "project", payload["project_name"])
|
||||
await _log_res(trace_id, "project", payload["project_name"], res)
|
||||
if res.resolved:
|
||||
c = res.candidates[0]
|
||||
payload["project_id"] = c.id
|
||||
payload["project_name"] = c.name
|
||||
resolved_projects[c.name] = c.id
|
||||
resolved_projects[payload["project_name"]] = c.id
|
||||
if payload["project_id"] not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(payload["project_id"])
|
||||
if c.id not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(c.id)
|
||||
options.pop("prj", None)
|
||||
elif res.candidates:
|
||||
options["prj"] = [{"id": c.id, "name": c.name} for c in res.candidates]
|
||||
@@ -131,21 +138,23 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
|
||||
for p_name in payload.get("project_names") or []:
|
||||
hit = matching.lookup(jev_map, p_name)
|
||||
if hit is not None:
|
||||
if hit["status"] == "exact":
|
||||
matching.apply_exact(payload, hit)
|
||||
continue
|
||||
res = await safe_resolve(ctx.resolver, "project", p_name)
|
||||
await _log_res(trace_id, "project", p_name, res)
|
||||
if res.resolved:
|
||||
c = res.candidates[0]
|
||||
p_id = c.id
|
||||
resolved_projects[c.name] = p_id
|
||||
resolved_projects[p_name] = p_id
|
||||
if p_id not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(p_id)
|
||||
if hit is not None and hit.get("status") == "exact":
|
||||
matching.apply_exact(payload, hit)
|
||||
resolved_projects[hit["name"]] = hit["id"]
|
||||
resolved_projects[p_name] = hit["id"]
|
||||
if hit["id"] not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(hit["id"])
|
||||
if not payload.get("project_id"):
|
||||
payload["project_id"] = p_id
|
||||
payload["project_id"] = hit["id"]
|
||||
elif hit is None:
|
||||
res = await safe_resolve(ctx.resolver, "project", p_name)
|
||||
if res.resolved:
|
||||
resolved_projects[p_name] = res.candidates[0].id
|
||||
resolved_projects[res.candidates[0].name] = res.candidates[0].id
|
||||
if res.candidates[0].id not in payload.get("project_ids", []):
|
||||
payload.setdefault("project_ids", []).append(res.candidates[0].id)
|
||||
if not payload.get("project_id"):
|
||||
payload["project_id"] = res.candidates[0].id
|
||||
|
||||
for i, item in enumerate(payload.get("items") or [], start=1):
|
||||
if not item.get("material_id") and item.get("material_name"):
|
||||
@@ -154,11 +163,15 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
|
||||
if hit["status"] == "exact":
|
||||
matching.apply_exact(payload, hit)
|
||||
options.pop(f"mat{i}", None)
|
||||
elif hit["status"] == "ambiguity" and hit.get("candidates"):
|
||||
options[f"mat{i}"] = [{"id": c["id"], "name": c["name"]} for c in hit["candidates"][:6]]
|
||||
elif hit["status"] == "new":
|
||||
options.pop(f"mat{i}", None)
|
||||
else:
|
||||
res = await safe_resolve(ctx.resolver, "material", item["material_name"])
|
||||
await _log_res(trace_id, "material", item["material_name"], res)
|
||||
if res.resolved:
|
||||
_set_material(item, res.candidates[0].id, res.candidates[0].name, res.candidates[0].extra)
|
||||
c = res.candidates[0]
|
||||
_set_material(item, c.id, c.name, c.extra)
|
||||
options.pop(f"mat{i}", None)
|
||||
elif res.candidates:
|
||||
options[f"mat{i}"] = [{"id": c.id, "name": c.name} for c in res.candidates]
|
||||
@@ -179,15 +192,13 @@ async def _resolve_unit(ctx, item: dict, trace_id: str | None = None,
|
||||
return
|
||||
if item.get("unit_name"):
|
||||
hit = matching.lookup(jev_map, item["unit_name"])
|
||||
if hit is not None:
|
||||
if hit["status"] == "exact":
|
||||
item["unit_id"] = hit["id"]
|
||||
item["unit_explicit"] = True
|
||||
item["unit_name"] = hit["name"]
|
||||
return
|
||||
else:
|
||||
if hit is not None and hit.get("status") == "exact":
|
||||
item["unit_id"] = hit["id"]
|
||||
item["unit_explicit"] = True
|
||||
item["unit_name"] = hit["name"]
|
||||
return
|
||||
if hit is None:
|
||||
res = await safe_resolve(ctx.resolver, "unit", item["unit_name"])
|
||||
await _log_res(trace_id, "unit", item["unit_name"], res)
|
||||
if res.resolved:
|
||||
item["unit_id"] = res.candidates[0].id
|
||||
item["unit_explicit"] = True
|
||||
@@ -369,46 +380,106 @@ async def commit(ctx, chat_id, user, draft, only_resolved: bool = False,
|
||||
|
||||
newly_created: dict = {}
|
||||
|
||||
# 1. Sequential pre-creation for new supplier if user wants to commit
|
||||
# 1. Defensive pre-creation for new supplier (get_or_create)
|
||||
if not payload.get("supplier_id") and payload.get("supplier_name"):
|
||||
cat_id = payload.get("supplier_category_id")
|
||||
if not cat_id:
|
||||
cats = await ctx.resolver.categories()
|
||||
if cats:
|
||||
cat_id = cats[0].id
|
||||
if cat_id:
|
||||
try:
|
||||
sup_body = {
|
||||
"name": payload["supplier_name"],
|
||||
"category_id": cat_id,
|
||||
}
|
||||
if payload.get("supplier_phone"):
|
||||
sup_body["phone"] = payload["supplier_phone"]
|
||||
created_sup = await ctx.api.create_supplier(sup_body, actor_id=user.actor_id)
|
||||
payload["supplier_id"] = created_sup["id"]
|
||||
newly_created["supplier"] = payload["supplier_name"]
|
||||
ctx.resolver.invalidate("suppliers")
|
||||
except Exception as e:
|
||||
logger.warning("Could not auto-create supplier: %s", e)
|
||||
sup_name = payload["supplier_name"].strip()
|
||||
# 1.a) Check if already in DB
|
||||
try:
|
||||
existing_sups = await ctx.api.list_suppliers(q=sup_name, page_size=10)
|
||||
for s in existing_sups:
|
||||
if normalize_text(s.get("name") or "") == normalize_text(sup_name):
|
||||
payload["supplier_id"] = str(s["id"])
|
||||
payload["supplier_name"] = s.get("name")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("Could not check existing suppliers: %s", e)
|
||||
|
||||
# 2. Sequential pre-creation for new materials
|
||||
# 1.b) If still not in DB, create it
|
||||
if not payload.get("supplier_id"):
|
||||
cat_id = payload.get("supplier_category_id")
|
||||
if not cat_id and payload.get("supplier_inferred_category_slug"):
|
||||
inferred_slug = payload["supplier_inferred_category_slug"]
|
||||
try:
|
||||
cats = await ctx.resolver.categories()
|
||||
for c in cats:
|
||||
if (c.extra or {}).get("slug") == inferred_slug or c.name == inferred_slug:
|
||||
cat_id = c.id
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if not cat_id:
|
||||
try:
|
||||
cats = await ctx.resolver.categories()
|
||||
if cats:
|
||||
cat_id = cats[0].id
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if cat_id:
|
||||
try:
|
||||
sup_body = {
|
||||
"name": sup_name,
|
||||
"category_id": cat_id,
|
||||
}
|
||||
if payload.get("supplier_phone"):
|
||||
sup_body["phone"] = payload["supplier_phone"]
|
||||
created_sup = await ctx.api.create_supplier(sup_body, actor_id=user.actor_id)
|
||||
payload["supplier_id"] = created_sup["id"]
|
||||
newly_created["supplier"] = sup_name
|
||||
ctx.resolver.invalidate("suppliers")
|
||||
except Exception as e:
|
||||
logger.warning("Could not auto-create supplier: %s — trying fallback fetch", e)
|
||||
try:
|
||||
existing_sups = await ctx.api.list_suppliers(q=sup_name, page_size=10)
|
||||
for s in existing_sups:
|
||||
if normalize_text(s.get("name") or "") == normalize_text(sup_name):
|
||||
payload["supplier_id"] = str(s["id"])
|
||||
payload["supplier_name"] = s.get("name")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Defensive pre-creation for new materials (get_or_create)
|
||||
created_mats = []
|
||||
for i, item in enumerate(payload.get("items") or [], start=1):
|
||||
if not item.get("material_id") and item.get("material_name") and item.get("unit_id"):
|
||||
mat_name = item["material_name"].strip()
|
||||
# 2.a) Check if already in DB
|
||||
try:
|
||||
created_mat = await ctx.api.create_material({
|
||||
"name": item["material_name"],
|
||||
"canonical_unit_id": int(item["unit_id"]),
|
||||
}, actor_id=user.actor_id)
|
||||
_set_material(item, created_mat["id"], created_mat.get("name") or item["material_name"])
|
||||
created_mats.append(item["material_name"])
|
||||
ctx.resolver.invalidate("materials")
|
||||
existing_mats = await ctx.api.list_materials(q=mat_name, page_size=10)
|
||||
for m in existing_mats:
|
||||
if normalize_text(m.get("name") or "") == normalize_text(mat_name):
|
||||
_set_material(item, str(m["id"]), m.get("name") or mat_name,
|
||||
{"canonical_unit_id": m.get("canonical_unit_id")})
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("Could not auto-create material: %s", e)
|
||||
logger.warning("Could not check existing materials: %s", e)
|
||||
|
||||
# 2.b) If still not in DB, create it
|
||||
if not item.get("material_id"):
|
||||
try:
|
||||
created_mat = await ctx.api.create_material({
|
||||
"name": mat_name,
|
||||
"canonical_unit_id": int(item["unit_id"]),
|
||||
}, actor_id=user.actor_id)
|
||||
_set_material(item, created_mat["id"], created_mat.get("name") or mat_name)
|
||||
created_mats.append(mat_name)
|
||||
ctx.resolver.invalidate("materials")
|
||||
except Exception as e:
|
||||
logger.warning("Could not auto-create material %s: %s — trying fallback fetch", mat_name, e)
|
||||
try:
|
||||
existing_mats = await ctx.api.list_materials(q=mat_name, page_size=10)
|
||||
for m in existing_mats:
|
||||
if normalize_text(m.get("name") or "") == normalize_text(mat_name):
|
||||
_set_material(item, str(m["id"]), m.get("name") or mat_name,
|
||||
{"canonical_unit_id": m.get("canonical_unit_id")})
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if created_mats:
|
||||
newly_created["materials"] = created_mats
|
||||
|
||||
# 3. Sequential pre-creation for new projects
|
||||
# 3. Defensive pre-creation for new projects (get_or_create)
|
||||
created_prjs = []
|
||||
all_p_names = []
|
||||
if payload.get("project_names"):
|
||||
@@ -429,14 +500,26 @@ async def commit(ctx, chat_id, user, draft, only_resolved: bool = False,
|
||||
all_p_names.append(it_p_str)
|
||||
|
||||
resolved_map = payload.setdefault("resolved_projects", {})
|
||||
all_known_prjs = None
|
||||
for p_name in all_p_names:
|
||||
p_id = resolved_map.get(p_name)
|
||||
if not p_id:
|
||||
res = await safe_resolve(ctx.resolver, "project", p_name)
|
||||
if res.resolved:
|
||||
p_id = res.candidates[0].id
|
||||
resolved_map[p_name] = p_id
|
||||
else:
|
||||
# 3.a) Check if project already exists in catalog
|
||||
if all_known_prjs is None:
|
||||
try:
|
||||
all_known_prjs = await ctx.resolver.projects()
|
||||
except Exception:
|
||||
all_known_prjs = []
|
||||
norm_pn = normalize_text(p_name)
|
||||
for prj in all_known_prjs:
|
||||
if normalize_text(prj.name) == norm_pn:
|
||||
p_id = prj.id
|
||||
resolved_map[p_name] = p_id
|
||||
resolved_map[prj.name] = p_id
|
||||
break
|
||||
|
||||
# 3.b) If not in catalog, create it
|
||||
if not p_id:
|
||||
try:
|
||||
created_prj = await ctx.api.create_project({"name": p_name}, actor_id=user.actor_id)
|
||||
p_id = created_prj.get("id")
|
||||
@@ -444,6 +527,7 @@ async def commit(ctx, chat_id, user, draft, only_resolved: bool = False,
|
||||
resolved_map[p_name] = p_id
|
||||
created_prjs.append(p_name)
|
||||
ctx.resolver.invalidate("projects")
|
||||
all_known_prjs = None
|
||||
except Exception as e:
|
||||
logger.warning("Could not auto-create project %s: %s", p_name, e)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from app.drafts import (
|
||||
KIND_SUPPLIER, merge_supplier_patch, new_supplier_payload, supplier_missing,
|
||||
)
|
||||
from app.keyboards import pick_buttons, simple_review_keyboard
|
||||
from app.resolve import safe_resolve
|
||||
from app.nlu import normalize_text
|
||||
from app.texts import ERR_GENERIC, esc, supplier_committed_report, supplier_review_text
|
||||
|
||||
logger = logging.getLogger("bot.supplier")
|
||||
@@ -33,17 +33,26 @@ async def handle_entities(ctx, chat_id, user, patch: dict, source: str | None,
|
||||
async def resolve_payload(ctx, payload: dict, trace_id: str | None = None) -> None:
|
||||
options = payload.setdefault("options", {})
|
||||
if not payload.get("category_id") and payload.get("category_name"):
|
||||
res = await safe_resolve(ctx.resolver, "category", payload["category_name"])
|
||||
await obs.log_resolution(trace_id, "category", payload["category_name"], res.status,
|
||||
[{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
|
||||
if res.resolved:
|
||||
payload["category_id"] = res.candidates[0].id
|
||||
payload["category_name"] = res.candidates[0].name
|
||||
norm_cat = normalize_text(payload["category_name"])
|
||||
try:
|
||||
cats = await ctx.resolver.categories()
|
||||
except Exception:
|
||||
cats = []
|
||||
exact = [c for c in cats if normalize_text(c.name) == norm_cat or (c.extra or {}).get("slug") == norm_cat]
|
||||
if len(exact) == 1:
|
||||
payload["category_id"] = exact[0].id
|
||||
payload["category_name"] = exact[0].name
|
||||
options.pop("cat", None)
|
||||
elif res.candidates:
|
||||
options["cat"] = [{"id": c.id, "name": c.name} for c in res.candidates]
|
||||
else:
|
||||
options.pop("cat", None)
|
||||
partial = [c for c in cats if norm_cat in normalize_text(c.name) and len(norm_cat) >= 3]
|
||||
if len(partial) == 1:
|
||||
payload["category_id"] = partial[0].id
|
||||
payload["category_name"] = partial[0].name
|
||||
options.pop("cat", None)
|
||||
elif partial:
|
||||
options["cat"] = [{"id": c.id, "name": c.name} for c in partial]
|
||||
else:
|
||||
options.pop("cat", None)
|
||||
|
||||
|
||||
async def apply_pick(ctx, chat_id, user, draft, field: str, choice: dict) -> None:
|
||||
|
||||
Reference in New Issue
Block a user