feat(matching): Jev entity matching phases 2-5 (per-field fan-out, variants, candidate ambiguity buttons)

Phase 2 — jev.py client: POST /v1/systemone, PINNED model jev-1.13.0 (drift warning when the response model differs), one retry on 429/5xx, fail-soft (None on any error), usage/latency returned for observability. Config adds TYPESAFE_API_KEY / JEV_* (thresholds 0.85/0.60, 15 questions per field); no daily cap by owner decision. bot/.env is now mounted read-only into the container so the key never enters compose/git. Phase 3 — matching.py: برون‌گروهی (one request per field, parallel), درون‌گروهی (one choice question per mention sharing the same full candidate list + «_none»), درون‌واریانسی (all STT spellings in one question). Routing: conf>=exact → bind; floor..exact → ambiguity; else/_none → new; missing/failed field → difflib fallback (bug fixed during testing: a failed request used to record 'new' instead of falling back). Resolver gains full cached lists for suppliers/materials (paged) plus brands (API+DB) and specs (EAV values + draft specs). NLU prompt now emits entity_mentions (6 fields) with the voice-variant grouping rule; sanitize_mentions guards the shape. handle_message runs matching only for register_quote/update_draft/add_material behind JEV_ENABLED; jev_map threads into quote/material resolve_payload (Jev hit skips difflib; exact writes ids; ambiguity skips difflib). Spec effect per owner: spelling normalization only (normalize_brand_spec). Phase 4 — entity-match ambiguity sub-messages: create_ambiguities() builds deduped entries {kind: entity_match, field, variants, candidates, details with Persian percentages}; new keyboards.ambiguity_candidates_keyboard renders one button per candidate + «هیچ‌کدام — ایجاد جدید»; callbacks handle amb⛏️{prefix}:{amb_id}:{idx} (applies the candidate via matching.apply_choice then re-resolves/renders) and amb:new (rejected → auto-create path at commit). Every user choice lands in observability (ambiguity.user_choice with arg+candidates). Phase 5 — voice variants: NLU groups different STT spellings of one entity into one variants[] (verified live: مصطفوی/مستفدی → one supplier mention); tools/test_matching.py live section (RUN_LIVE=1) exercises real DeepSeek + real Jev without touching Bale. Tests: test_jev.py 5/5 (mock HTTP: no-key, happy path, 429-retry, 400, exception), test_matching.py 11/11 mock + 3/3 live (real NLU produced 7 mentions with correct fields; real Jev matched «50 کیلویی»→«۵۰ کیلویی» 0.98 and «کیسه» 1.0 and correctly returned new against the emptied catalog); observability regression 8/8.
2026-10-12 21:24:03 +03:30
parent d58b619107
commit 1c853e50a9
16 changed files with 1029 additions and 70 deletions
@@ -224,7 +224,20 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
    result = await understand(settings, input_payload, draft_summary, trace_id=trace_id)
    intent, patch = result.get("intent"), result.get("patch") or {}
    consensus_text = result.get("consensus_text") or text
-    await obs.event(trace_id, "entities.extracted", payload={"patch": patch, "intent": intent})
+    mentions = result.get("entity_mentions") or []
+    await obs.event(trace_id, "entities.extracted",
+                    payload={"patch": patch, "intent": intent, "entity_mentions": mentions})
+
+    # Phase 3: Jev per-field matching — only when it can be applied (flag-gated;
+    # a failure leaves jev_map empty and the difflib path runs unchanged).
+    jev_map: dict = {}
+    if settings.JEV_ENABLED and mentions and intent in ("register_quote", "update_draft", "add_material"):
+        try:
+            from app import matching
+            jev_map = await matching.resolve_mentions(ctx, patch, mentions, trace_id)
+        except Exception:
+            logger.exception("jev matching failed")
+            jev_map = {}

    if source == "voice":
        await ctx._log(chat_id, "in", "voice", consensus_text)
@@ -238,6 +251,7 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
            incoming_msg_id=message_id,
            target_draft=target_draft,
            trace_id=trace_id,
+            jev_map=jev_map,
        )
    elif intent == "commit_draft":
        active = target_draft or await draft_store.get_active(chat_id)
@@ -280,7 +294,7 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
    elif intent == "add_supplier":
        await supplier.handle_entities(ctx, chat_id, user, patch, source, trace_id=trace_id)
    elif intent == "add_material":
-        await material.handle_entities(ctx, chat_id, user, patch, source, trace_id=trace_id)
+        await material.handle_entities(ctx, chat_id, user, patch, source, trace_id=trace_id, jev_map=jev_map)
    elif intent == "add_project":
        await project.handle_entities(ctx, chat_id, user, patch, source, trace_id=trace_id)
    elif intent == "status_query":
@@ -82,6 +82,8 @@ async def _dispatch(ctx, chat_id: int, data: str, cq: dict) -> None:
        await obs.event(tid, "ambiguity.user_choice", payload={
            "amb_id": amb_id, "action": action, "kind": target_amb.get("kind"),
            "title": target_amb.get("title"),
+            "arg": parts[4] if len(parts) > 4 else None,
+            "candidates": target_amb.get("candidates"),
        })

        if action == "ok":
@@ -107,6 +109,38 @@ async def _dispatch(ctx, chat_id: int, data: str, cq: dict) -> None:
                    await ctx.delete(chat_id, target_amb["message_id"])
                except Exception:
                    pass
+        elif action == "pick" and len(parts) >= 5:
+            # Phase 4: user picked one of the Jev candidates.
+            try:
+                idx = int(parts[4])
+            except ValueError:
+                return
+            cands = target_amb.get("candidates") or []
+            if not (0 <= idx < len(cands)):
+                return
+            from app import matching
+            matching.apply_choice(payload, target_amb.get("field"), cands[idx],
+                                  target_amb.get("variants") or [])
+            target_amb["status"] = "applied"
+            await quote.resolve_payload(ctx, payload)
+            await draft_store.save_payload(draft, mark_bale_synced=True)
+            # In-place edit main review card
+            await quote.render(ctx, chat_id, draft, fresh=False)
+            # Delete ONLY this ambiguity message
+            if target_amb.get("message_id"):
+                try:
+                    await ctx.delete(chat_id, target_amb["message_id"])
+                except Exception:
+                    pass
+        elif action == "new":
+            # «هیچ‌کدام» — keep the raw name; the auto-create path handles it at commit.
+            target_amb["status"] = "rejected"
+            await draft_store.save_payload(draft, mark_bale_synced=True)
+            if target_amb.get("message_id"):
+                try:
+                    await ctx.delete(chat_id, target_amb["message_id"])
+                except Exception:
+                    pass
        return

    # ---- pick a candidate for a draft field -------------------------------
@@ -2,6 +2,7 @@
 import logging

 from app import drafts as draft_store
+from app import matching
 from app import observability as obs
 from app.api.client import ApiError
 from app.drafts import (
@@ -15,7 +16,8 @@ logger = logging.getLogger("bot.material")


 async def handle_entities(ctx, chat_id, user, patch: dict, source: str | None,
-                          trace_id: str | None = None) -> None:
+                          trace_id: str | None = None,
+                          jev_map: dict | None = None) -> None:
    draft = await draft_store.get_active(chat_id, KIND_MATERIAL)
    if draft is not None:
        payload = draft.payload
@@ -25,25 +27,36 @@ async def handle_entities(ctx, chat_id, user, patch: dict, source: str | None,
        merge_material_patch(payload, patch)
        draft = await draft_store.create(chat_id, KIND_MATERIAL, payload, source=source)
    await obs.link_draft(trace_id, draft.id, draft.kind)
-    await resolve_payload(ctx, payload, trace_id)
+    await resolve_payload(ctx, payload, trace_id, jev_map)
    await draft_store.save_payload(draft, missing={"fields": material_missing(payload)})
    await render(ctx, chat_id, draft, fresh=draft.review_message_id is None, trace_id=trace_id)


-async def resolve_payload(ctx, payload: dict, trace_id: str | None = None) -> None:
+async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
+                          jev_map: dict | None = None) -> None:
    options = payload.setdefault("options", {})
    if not payload.get("canonical_unit_id") and payload.get("unit_name"):
-        res = await safe_resolve(ctx.resolver, "unit", payload["unit_name"])
-        await obs.log_resolution(trace_id, "unit", payload["unit_name"], res.status,
-                                 [{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
-        if res.resolved:
-            payload["canonical_unit_id"] = res.candidates[0].id
-            payload["unit_name"] = res.candidates[0].name
-            options.pop("unit", None)
-        elif res.candidates:
-            options["unit"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+        hit = matching.lookup(jev_map, payload["unit_name"])
+        if hit is not None:
+            if hit["status"] == "exact":
+                payload["canonical_unit_id"] = hit["id"]
+                payload["unit_name"] = hit["name"]
+                options.pop("unit", None)
+            elif hit["status"] == "ambiguity" and hit.get("candidates"):
+                options["unit"] = [{"id": c["id"], "name": c["name"]}
+                                   for c in hit["candidates"][:6]]
        else:
-            options.pop("unit", None)
+            res = await safe_resolve(ctx.resolver, "unit", payload["unit_name"])
+            await obs.log_resolution(trace_id, "unit", payload["unit_name"], res.status,
+                                     [{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])
+            if res.resolved:
+                payload["canonical_unit_id"] = res.candidates[0].id
+                payload["unit_name"] = res.candidates[0].name
+                options.pop("unit", None)
+            elif res.candidates:
+                options["unit"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+            else:
+                options.pop("unit", None)


 async def apply_pick(ctx, chat_id, user, draft, field: str, choice: dict) -> None:
@@ -105,5 +118,5 @@ async def commit(ctx, chat_id, user, draft, trace_id: str | None = None) -> None
    await obs.event(trace_id, "draft.commit", payload={"kind": "material",
                                                       "material_id": (created or {}).get("id")})
    await obs.set_trace_status(trace_id, "committed")
-    ctx.resolver.invalidate("materials")
+    ctx.resolver.invalidate()
    await ctx.send(chat_id, material_committed_report(created))
@@ -72,5 +72,5 @@ async def commit(ctx, chat_id, user, draft, trace_id: str | None = None) -> None
    await obs.event(trace_id, "draft.commit", payload={"kind": "project",
                                                       "project_id": (created or {}).get("id")})
    await obs.set_trace_status(trace_id, "committed")
-    ctx.resolver.invalidate("projects")
+    ctx.resolver.invalidate()
    await ctx.send(chat_id, project_committed_report(created))
@@ -2,6 +2,7 @@
 import logging

 from app import drafts as draft_store
+from app import matching
 from app import observability as obs
 from app.api.client import ApiError
 from app.drafts import (
@@ -22,6 +23,7 @@ async def handle_entities(
    incoming_msg_id: int | None = None,
    target_draft = None,
    trace_id: str | None = None,
+    jev_map: dict | None = None,
 ) -> None:
    prev_review_msg_id = None
    if target_draft is not None:
@@ -50,7 +52,10 @@ async def handle_entities(
        )

    await obs.link_draft(trace_id, draft.id, draft.kind)
-    await resolve_payload(ctx, payload, trace_id)
+    await resolve_payload(ctx, payload, trace_id, jev_map)
+    if jev_map:
+        matching.normalize_brand_spec(payload, jev_map)
+        matching.create_ambiguities(payload, jev_map)

    if prev_review_msg_id:
        try:
@@ -71,46 +76,65 @@ async def _log_res(trace_id, field: str, mention, res) -> None:
                             [{"id": c.id, "name": c.name} for c in (res.candidates or [])[:4]])


-async def resolve_payload(ctx, payload: dict, trace_id: str | None = None) -> None:
+async def resolve_payload(ctx, payload: dict, trace_id: str | None = None,
+                          jev_map: dict | None = None) -> None:
    options = payload.setdefault("options", {})

    if not payload.get("supplier_id") and payload.get("supplier_name"):
-        res = await safe_resolve(ctx.resolver, "supplier", payload["supplier_name"])
-        await _log_res(trace_id, "supplier", payload["supplier_name"], res)
-        if res.resolved:
-            c = res.candidates[0]
-            payload["supplier_id"] = c.id
-            payload["supplier_name"] = c.name
-            extra = c.extra or {}
-            for k in ("location", "phone", "contact_person", "category_name"):
-                if extra.get(k):
-                    payload[f"supplier_{k}"] = extra[k]
-            options.pop("sup", None)
-        elif res.candidates:
-            options["sup"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+        hit = matching.lookup(jev_map, payload["supplier_name"])
+        if hit is not None:
+            if hit["status"] == "exact":
+                matching.apply_exact(payload, hit)
+                options.pop("sup", None)
        else:
-            options.pop("sup", None)
+            res = await safe_resolve(ctx.resolver, "supplier", payload["supplier_name"])
+            await _log_res(trace_id, "supplier", payload["supplier_name"], res)
+            if res.resolved:
+                c = res.candidates[0]
+                payload["supplier_id"] = c.id
+                payload["supplier_name"] = c.name
+                extra = c.extra or {}
+                for k in ("location", "phone", "contact_person", "category_name"):
+                    if extra.get(k):
+                        payload[f"supplier_{k}"] = extra[k]
+                options.pop("sup", None)
+            elif res.candidates:
+                options["sup"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+            else:
+                options.pop("sup", None)

    resolved_projects = payload.setdefault("resolved_projects", {})

    if not payload.get("project_id") and payload.get("project_name"):
-        res = await safe_resolve(ctx.resolver, "project", payload["project_name"])
-        await _log_res(trace_id, "project", payload["project_name"], res)
-        if res.resolved:
-            c = res.candidates[0]
-            payload["project_id"] = c.id
-            payload["project_name"] = c.name
-            resolved_projects[c.name] = c.id
-            resolved_projects[payload["project_name"]] = c.id
-            if payload["project_id"] not in payload.get("project_ids", []):
-                payload.setdefault("project_ids", []).append(payload["project_id"])
-            options.pop("prj", None)
-        elif res.candidates:
-            options["prj"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+        hit = matching.lookup(jev_map, payload["project_name"])
+        if hit is not None:
+            if hit["status"] == "exact":
+                matching.apply_exact(payload, hit)
+                payload["project_name"] = hit["name"]
+                options.pop("prj", None)
        else:
-            options.pop("prj", None)
+            res = await safe_resolve(ctx.resolver, "project", payload["project_name"])
+            await _log_res(trace_id, "project", payload["project_name"], res)
+            if res.resolved:
+                c = res.candidates[0]
+                payload["project_id"] = c.id
+                payload["project_name"] = c.name
+                resolved_projects[c.name] = c.id
+                resolved_projects[payload["project_name"]] = c.id
+                if payload["project_id"] not in payload.get("project_ids", []):
+                    payload.setdefault("project_ids", []).append(payload["project_id"])
+                options.pop("prj", None)
+            elif res.candidates:
+                options["prj"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+            else:
+                options.pop("prj", None)

    for p_name in payload.get("project_names") or []:
+        hit = matching.lookup(jev_map, p_name)
+        if hit is not None:
+            if hit["status"] == "exact":
+                matching.apply_exact(payload, hit)
+            continue
        res = await safe_resolve(ctx.resolver, "project", p_name)
        await _log_res(trace_id, "project", p_name, res)
        if res.resolved:
@@ -125,17 +149,23 @@ async def resolve_payload(ctx, payload: dict, trace_id: str | None = None) -> No

    for i, item in enumerate(payload.get("items") or [], start=1):
        if not item.get("material_id") and item.get("material_name"):
-            res = await safe_resolve(ctx.resolver, "material", item["material_name"])
-            await _log_res(trace_id, "material", item["material_name"], res)
-            if res.resolved:
-                _set_material(item, res.candidates[0].id, res.candidates[0].name, res.candidates[0].extra)
-                options.pop(f"mat{i}", None)
-            elif res.candidates:
-                options[f"mat{i}"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+            hit = matching.lookup(jev_map, item["material_name"])
+            if hit is not None:
+                if hit["status"] == "exact":
+                    matching.apply_exact(payload, hit)
+                    options.pop(f"mat{i}", None)
            else:
-                options.pop(f"mat{i}", None)
+                res = await safe_resolve(ctx.resolver, "material", item["material_name"])
+                await _log_res(trace_id, "material", item["material_name"], res)
+                if res.resolved:
+                    _set_material(item, res.candidates[0].id, res.candidates[0].name, res.candidates[0].extra)
+                    options.pop(f"mat{i}", None)
+                elif res.candidates:
+                    options[f"mat{i}"] = [{"id": c.id, "name": c.name} for c in res.candidates]
+                else:
+                    options.pop(f"mat{i}", None)
        if not item.get("unit_id"):
-            await _resolve_unit(ctx, item, trace_id)
+            await _resolve_unit(ctx, item, trace_id, jev_map)


 def _set_material(item: dict, material_id: str, name: str, extra: dict | None = None) -> None:
@@ -143,17 +173,26 @@ def _set_material(item: dict, material_id: str, name: str, extra: dict | None =
    item["material_row"] = {"id": material_id, "canonical_unit_id": (extra or {}).get("canonical_unit_id")}


-async def _resolve_unit(ctx, item: dict, trace_id: str | None = None) -> None:
+async def _resolve_unit(ctx, item: dict, trace_id: str | None = None,
+                        jev_map: dict | None = None) -> None:
    if item.get("unit_id"):
        return
    if item.get("unit_name"):
-        res = await safe_resolve(ctx.resolver, "unit", item["unit_name"])
-        await _log_res(trace_id, "unit", item["unit_name"], res)
-        if res.resolved:
-            item["unit_id"] = res.candidates[0].id
-            item["unit_explicit"] = True
-            item["unit_name"] = res.candidates[0].name
-            return
+        hit = matching.lookup(jev_map, item["unit_name"])
+        if hit is not None:
+            if hit["status"] == "exact":
+                item["unit_id"] = hit["id"]
+                item["unit_explicit"] = True
+                item["unit_name"] = hit["name"]
+                return
+        else:
+            res = await safe_resolve(ctx.resolver, "unit", item["unit_name"])
+            await _log_res(trace_id, "unit", item["unit_name"], res)
+            if res.resolved:
+                item["unit_id"] = res.candidates[0].id
+                item["unit_explicit"] = True
+                item["unit_name"] = res.candidates[0].name
+                return
    canonical = ((item.get("material_row") or {}).get("canonical_unit_id"))
    if canonical:
        item["unit_id"] = str(canonical)
@@ -110,5 +110,5 @@ async def commit(ctx, chat_id, user, draft, trace_id: str | None = None) -> None
    await obs.event(trace_id, "draft.commit", payload={"kind": "supplier",
                                                       "supplier_id": (created or {}).get("id")})
    await obs.set_trace_status(trace_id, "committed")
-    ctx.resolver.invalidate("suppliers")
+    ctx.resolver.invalidate()
    await ctx.send(chat_id, supplier_committed_report(created))