"""Phase 3-5 tests for entity matching (mock by default; live with RUN_LIVE=1). docker compose exec bot python tools/test_matching.py # mock only docker compose exec -e RUN_LIVE=1 bot python tools/test_matching.py # + real NLU + Jev The live section calls DeepSeek (NLU) and TypeSafe (Jev) with the REAL keys from bot/.env — it never sends anything to Bale and never creates drafts. """ import asyncio import os import sys from types import SimpleNamespace from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from app import matching from app.config import settings from app.nlu import normalize_text from app.resolve import Candidate PASSED: list[str] = [] FAILED: list[str] = [] def check(name: str, ok: bool, extra: str = "") -> None: (PASSED if ok else FAILED).append(name) print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}") class FakeResolver: async def projects(self): return [Candidate(id="p1", name="دفتر"), Candidate(id="p2", name="منشانه")] async def all_suppliers(self): return [Candidate(id="s1", name="بازرگانی مصطفوی", extra={"location": "اصفهان", "category_name": "مصالح"})] async def all_materials(self): return [Candidate(id="m1", name="گچ و خاک", extra={"canonical_unit_id": 2})] async def brands(self): return [Candidate(id="b1", name="سیمان تهران")] async def specs(self): return [Candidate(id="سایز ۱۴", name="سایز ۱۴")] async def units(self): return [Candidate(id="u1", name="کیسه"), Candidate(id="u2", name="عدد")] def make_fake_ask(answers_by_mention: dict, model: str = "jev-1.13.0"): async def fake_ask(state, questions): if fake_ask.fail: return None return {"answers": answers_by_mention, "usage": {"input_tokens": 1, "output_tokens": 1}, "model": model, "latency_ms": 7} fake_ask.fail = False return fake_ask async def mock_tests() -> None: ctx = SimpleNamespace(resolver=FakeResolver()) orig_ask = matching.jev_ask # 1) routing bands: exact / ambiguity / low-trust / new + variant grouping matching.jev_ask = make_fake_ask({ "mention_0": {"choice": "منشانه", "confidence": 0.9, "probabilities": {"منشانه": 0.95, "_none": 0.05}}, "mention_1": {"choice": "دفتر", "confidence": 0.7, "probabilities": {"دفتر": 0.73, "منشانه": 0.15, "_none": 0.12}}, "mention_2": {"choice": "_none", "confidence": 0.9, "probabilities": {"_none": 0.94}}, "mention_3": {"choice": "دفتر", "confidence": 0.5, "probabilities": {"دفتر": 0.52, "_none": 0.48}}, }) orig_fallback = matching.match_with_deepseek async def mock_fallback_below_floor(*args, **kwargs): ms = kwargs.get("mentions") or (args[1] if len(args) > 1 else []) return {normalize_text(v): {"status": "new", "field": m.get("field"), "variants": m.get("variants"), "id": None} for m in ms for v in m.get("variants", [])} matching.match_with_deepseek = mock_fallback_below_floor orig_arbiter = matching.adjudicate_with_deepseek async def mock_arbiter(*args, **kwargs): items = kwargs.get("items") or (args[2] if len(args) > 2 else []) res = {} for it in items: choice = it.get("jev_choice") conf = it.get("jev_confidence", 0.0) if choice == "منشانه" and conf >= 0.8: entry = {"status": "exact", "id": "p2", "name": "منشانه", "field": it["field"], "variants": it["variants"]} elif choice == "دفتر" and conf >= 0.6: entry = {"status": "ambiguity", "id": "p1", "name": "دفتر", "field": it["field"], "variants": it["variants"], "candidates": [{"id": "p1", "name": "دفتر", "prob": conf}]} else: entry = {"status": "new", "id": None, "name": None, "field": it["field"], "variants": it["variants"], "candidates": []} for v in it["variants"]: res[normalize_text(v)] = entry return res matching.adjudicate_with_deepseek = mock_arbiter mentions = [ {"field": "project", "variants": ["منشیانه"]}, {"field": "project", "variants": ["دفتر", "دفر"]}, {"field": "project", "variants": ["چیز عجیب"]}, {"field": "project", "variants": ["دفر دو"]}, ] try: res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None) finally: matching.match_with_deepseek = orig_fallback matching.adjudicate_with_deepseek = orig_arbiter r0 = matching.lookup(res, "منشیانه") r1 = matching.lookup(res, "دفر") # variant key resolves too r2 = matching.lookup(res, "چیز عجیب") check("exact band -> status exact + id", r0 and r0["status"] == "exact" and r0["id"] == "p2") check("mid band -> ambiguity + candidates", r1 and r1["status"] == "ambiguity" and r1["candidates"][0]["name"] == "دفتر") check("variant of same mention shares one result", r1 is r0 if False else (r1 and matching.lookup(res, "دفتر") is r1)) check("_none / low conf -> new", r2 and r2["status"] == "new") check("below-floor pick -> routed to crisis fallback", matching.lookup(res, "دفر دو") and matching.lookup(res, "دفر دو")["status"] == "new") check("below-floor pick did not pollute other mentions", r1 and r1["status"] == "ambiguity") # 2) Jev failure -> DeepSeek crisis fallback matching.jev_ask = make_fake_ask({}) matching.jev_ask.fail = True async def mock_deepseek(*args, **kwargs): return {normalize_text("منشیانه"): {"status": "exact", "id": "p2", "name": "منشانه", "field": "project", "variants": ["منشیانه"]}} matching.match_with_deepseek = mock_deepseek try: res_fail = await matching.resolve_mentions(ctx, {"items": []}, mentions, None) check("jev failure -> triggers crisis fallback", normalize_text("منشیانه") in res_fail and res_fail[normalize_text("منشیانه")]["status"] == "exact") finally: matching.match_with_deepseek = orig_fallback # 3) apply_exact writes supplier/project ids into a payload payload = {"items": []} matching.apply_exact(payload, {"field": "project", "id": "p2", "name": "منشانه", "variants": ["منشیانه"], "scores": {}}) matching.apply_exact(payload, {"field": "supplier", "id": "s1", "name": "بازرگانی مصطفوی", "extra": {"location": "اصفهان"}, "variants": ["مستفدی"]}) check("apply_exact -> project_id/resolved_projects/project_ids", payload.get("project_id") == "p2" and payload["resolved_projects"].get("منشیانه") == "p2") check("apply_exact -> supplier_id/name/extra", payload.get("supplier_id") == "s1" and payload.get("supplier_location") == "اصفهان") # 4) material/unit/brand/spec application on items payload2 = {"items": [{"material_name": "مصطفی", "unit_name": "کیسه", "brand_name": "سیمان تهران", "specs": ["سایز 14"]}]} matching.apply_choice(payload2, "material", {"id": "m1", "name": "گچ و خاک", "extra": {"canonical_unit_id": 2}}, ["مصطفی"]) matching.apply_choice(payload2, "unit", {"id": "u1", "name": "کیسه"}, ["کیسه"]) matching.normalize_brand_spec(payload2, {normalize_text("سایز 14"): { "field": "spec", "status": "exact", "name": "سایز ۱۴", "id": "سایز ۱۴", "variants": ["سایز 14"]}}) it = payload2["items"][0] check("material/unit applied on matching items", it["material_id"] == "m1" and it["unit_id"] == "u1" and it["unit_explicit"] is True) check("spec normalization (spelling only)", it["specs"][0] == "سایز ۱۴") # 5) create_ambiguities builds an entity_match entry with percentages payload3 = {"items": [], "ambiguities": []} jev_map = {normalize_text("منشیانه"): { "field": "project", "status": "ambiguity", "variants": ["منشیانه"], "candidates": [{"id": "p2", "name": "منشانه", "prob": 0.8}, {"id": "p1", "name": "دفتر", "prob": 0.02}]}} matching.create_ambiguities(payload3, jev_map) ambs = payload3["ambiguities"] check("entity_match ambiguity created with candidates + details", len(ambs) == 1 and ambs[0]["kind"] == "entity_match" and len(ambs[0]["candidates"]) == 2 and "۸۰٪" in ambs[0]["details"]) matching.create_ambiguities(payload3, jev_map) # idempotent check("create_ambiguities is deduped", len(payload3["ambiguities"]) == 1) matching.jev_ask = orig_ask # --------------------------------------------------------------------------- # live section (real NLU + real Jev) — opt-in and Bale-free # --------------------------------------------------------------------------- HARNESS_TRANSCRIPT = ( "مدل 1 (Whisper): الان شبت بخیر می خواستیم که بیست و پنج تا کیسه گچ و خاک از مصطفوی بخریم " "و کنارش هم یه پنجاه تا کیسه سیمان پنجاه کیلویی. خریدمون هم عمدتا برای منشیانه و دفتر. " "پیش فاکتورم گفته تا حدودا بیست روزی اوکیه.\n" "مدل 2 (Vosk): میخواستیم 25 تا کیسه گچ و خاک از مستفدی بخریم و کنارشم یه 50 تا کیسه سیمان " "50 کیلویی، خریدمون هم برای منشیانه و دفتر." ) async def live_tests() -> None: from app.api.client import ErpClient from app.resolve import Resolver from app.nlu import understand api = ErpClient(settings.API_BASE_URL, settings.API_TIMEOUT_SECONDS, settings.SERVICE_TOKEN) ctx = SimpleNamespace(resolver=Resolver(api)) print("\n--- LIVE: NLU entity_mentions ---") result = await understand(settings, HARNESS_TRANSCRIPT) mentions = result.get("entity_mentions") or [] print("intent:", result.get("intent"), "| mentions:", mentions) check("live NLU produced entity_mentions", len(mentions) >= 2, f"{len(mentions)} mentions") sup = next((m for m in mentions if m["field"] == "supplier"), None) check("voice spellings grouped into ONE supplier mention", bool(sup) and len(sup["variants"]) >= 2, str(sup)) print("\n--- LIVE: Jev per-field matching (real data) ---") jev_map = await matching.resolve_mentions(ctx, result.get("patch") or {}, mentions, None) for key, r in sorted(jev_map.items()): print(f" {key!r}: {r['status']} -> {r.get('name')!r} conf={r.get('confidence')}") check("live matching returned results", len(jev_map) > 0, f"{len(jev_map)} variants") async def main() -> None: await mock_tests() if os.environ.get("RUN_LIVE") == "1": if not settings.TYPESAFE_API_KEY: check("live section requires TYPESAFE_API_KEY", False, "bot/.env not mounted?") else: await live_tests() else: print("\n(live section skipped — run with RUN_LIVE=1 to hit real NLU + Jev)") total = len(PASSED) + len(FAILED) print(f"\n{len(PASSED)}/{total} passed") if FAILED: print("FAILED:", FAILED) raise SystemExit(1) if __name__ == "__main__": asyncio.run(main())