whqreza
9e848f535f
- Add app/fallback_matcher.py for intelligent crisis entity matching with token and latency observability - Route Jev failure/timeouts in app/matching.py to DeepSeek fallback engine - Handle MatchingFailureError defensively in handlers/__init__.py with honest error message - Enforce strict 'new' entity status in handlers/quote.py and implement defensive get_or_create on commit - Clean up app/resolve.py to eliminate difflib and character ratio heuristics - Pass all smoke tests (59/59), fallback matching tests (15/15), and matching unit tests (13/13)
222 lines
10 KiB
Python
222 lines
10 KiB
Python
"""Phase 3-5 tests for entity matching (mock by default; live with RUN_LIVE=1).
|
|
|
|
docker compose exec bot python tools/test_matching.py # mock only
|
|
docker compose exec -e RUN_LIVE=1 bot python tools/test_matching.py # + real NLU + Jev
|
|
|
|
The live section calls DeepSeek (NLU) and TypeSafe (Jev) with the REAL keys
|
|
from bot/.env — it never sends anything to Bale and never creates drafts.
|
|
"""
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
from types import SimpleNamespace
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app import matching
|
|
from app.config import settings
|
|
from app.nlu import normalize_text
|
|
from app.resolve import Candidate
|
|
|
|
PASSED: list[str] = []
|
|
FAILED: list[str] = []
|
|
|
|
|
|
def check(name: str, ok: bool, extra: str = "") -> None:
|
|
(PASSED if ok else FAILED).append(name)
|
|
print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}")
|
|
|
|
|
|
class FakeResolver:
|
|
async def projects(self):
|
|
return [Candidate(id="p1", name="دفتر"), Candidate(id="p2", name="منشانه")]
|
|
|
|
async def all_suppliers(self):
|
|
return [Candidate(id="s1", name="بازرگانی مصطفوی",
|
|
extra={"location": "اصفهان", "category_name": "مصالح"})]
|
|
|
|
async def all_materials(self):
|
|
return [Candidate(id="m1", name="گچ و خاک", extra={"canonical_unit_id": 2})]
|
|
|
|
async def brands(self):
|
|
return [Candidate(id="b1", name="سیمان تهران")]
|
|
|
|
async def specs(self):
|
|
return [Candidate(id="سایز ۱۴", name="سایز ۱۴")]
|
|
|
|
async def units(self):
|
|
return [Candidate(id="u1", name="کیسه"), Candidate(id="u2", name="عدد")]
|
|
|
|
|
|
def make_fake_ask(answers_by_mention: dict, model: str = "jev-1.13.0"):
|
|
async def fake_ask(state, questions):
|
|
if fake_ask.fail:
|
|
return None
|
|
return {"answers": answers_by_mention, "usage": {"input_tokens": 1, "output_tokens": 1},
|
|
"model": model, "latency_ms": 7}
|
|
fake_ask.fail = False
|
|
return fake_ask
|
|
|
|
|
|
async def mock_tests() -> None:
|
|
ctx = SimpleNamespace(resolver=FakeResolver())
|
|
orig_ask = matching.jev_ask
|
|
|
|
# 1) routing bands: exact / ambiguity / low-trust / new + variant grouping
|
|
matching.jev_ask = make_fake_ask({
|
|
"mention_0": {"choice": "منشانه", "confidence": 0.9,
|
|
"probabilities": {"منشانه": 0.95, "_none": 0.05}},
|
|
"mention_1": {"choice": "دفتر", "confidence": 0.7,
|
|
"probabilities": {"دفتر": 0.73, "منشانه": 0.15, "_none": 0.12}},
|
|
"mention_2": {"choice": "_none", "confidence": 0.9,
|
|
"probabilities": {"_none": 0.94}},
|
|
"mention_3": {"choice": "دفتر", "confidence": 0.5,
|
|
"probabilities": {"دفتر": 0.52, "_none": 0.48}},
|
|
})
|
|
orig_fallback = matching.match_with_deepseek
|
|
async def mock_fallback_below_floor(*args, **kwargs):
|
|
ms = kwargs.get("mentions") or (args[1] if len(args) > 1 else [])
|
|
return {normalize_text(v): {"status": "new", "field": m.get("field"), "variants": m.get("variants"), "id": None}
|
|
for m in ms for v in m.get("variants", [])}
|
|
matching.match_with_deepseek = mock_fallback_below_floor
|
|
|
|
mentions = [
|
|
{"field": "project", "variants": ["منشیانه"]},
|
|
{"field": "project", "variants": ["دفتر", "دفر"]},
|
|
{"field": "project", "variants": ["چیز عجیب"]},
|
|
{"field": "project", "variants": ["دفر دو"]},
|
|
]
|
|
try:
|
|
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
|
finally:
|
|
matching.match_with_deepseek = orig_fallback
|
|
|
|
r0 = matching.lookup(res, "منشیانه")
|
|
r1 = matching.lookup(res, "دفر") # variant key resolves too
|
|
r2 = matching.lookup(res, "چیز عجیب")
|
|
check("exact band -> status exact + id", r0 and r0["status"] == "exact" and r0["id"] == "p2")
|
|
check("mid band -> ambiguity + candidates", r1 and r1["status"] == "ambiguity"
|
|
and r1["candidates"][0]["name"] == "دفتر")
|
|
check("variant of same mention shares one result", r1 is r0 if False else
|
|
(r1 and matching.lookup(res, "دفتر") is r1))
|
|
check("_none / low conf -> new", r2 and r2["status"] == "new")
|
|
check("below-floor pick -> routed to crisis fallback",
|
|
matching.lookup(res, "دفر دو") and matching.lookup(res, "دفر دو")["status"] == "new")
|
|
check("below-floor pick did not pollute other mentions", r1 and r1["status"] == "ambiguity")
|
|
|
|
# 2) Jev failure -> DeepSeek crisis fallback
|
|
matching.jev_ask = make_fake_ask({})
|
|
matching.jev_ask.fail = True
|
|
async def mock_deepseek(*args, **kwargs):
|
|
return {normalize_text("منشیانه"): {"status": "exact", "id": "p2", "name": "منشانه",
|
|
"field": "project", "variants": ["منشیانه"]}}
|
|
matching.match_with_deepseek = mock_deepseek
|
|
try:
|
|
res_fail = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
|
check("jev failure -> triggers crisis fallback",
|
|
normalize_text("منشیانه") in res_fail and res_fail[normalize_text("منشیانه")]["status"] == "exact")
|
|
finally:
|
|
matching.match_with_deepseek = orig_fallback
|
|
|
|
# 3) apply_exact writes supplier/project ids into a payload
|
|
payload = {"items": []}
|
|
matching.apply_exact(payload, {"field": "project", "id": "p2", "name": "منشانه",
|
|
"variants": ["منشیانه"], "scores": {}})
|
|
matching.apply_exact(payload, {"field": "supplier", "id": "s1", "name": "بازرگانی مصطفوی",
|
|
"extra": {"location": "اصفهان"}, "variants": ["مستفدی"]})
|
|
check("apply_exact -> project_id/resolved_projects/project_ids",
|
|
payload.get("project_id") == "p2"
|
|
and payload["resolved_projects"].get("منشیانه") == "p2")
|
|
check("apply_exact -> supplier_id/name/extra",
|
|
payload.get("supplier_id") == "s1" and payload.get("supplier_location") == "اصفهان")
|
|
|
|
# 4) material/unit/brand/spec application on items
|
|
payload2 = {"items": [{"material_name": "مصطفی", "unit_name": "کیسه",
|
|
"brand_name": "سیمان تهران", "specs": ["سایز 14"]}]}
|
|
matching.apply_choice(payload2, "material", {"id": "m1", "name": "گچ و خاک",
|
|
"extra": {"canonical_unit_id": 2}},
|
|
["مصطفی"])
|
|
matching.apply_choice(payload2, "unit", {"id": "u1", "name": "کیسه"}, ["کیسه"])
|
|
matching.normalize_brand_spec(payload2, {normalize_text("سایز 14"): {
|
|
"field": "spec", "status": "exact", "name": "سایز ۱۴", "id": "سایز ۱۴",
|
|
"variants": ["سایز 14"]}})
|
|
it = payload2["items"][0]
|
|
check("material/unit applied on matching items",
|
|
it["material_id"] == "m1" and it["unit_id"] == "u1" and it["unit_explicit"] is True)
|
|
check("spec normalization (spelling only)", it["specs"][0] == "سایز ۱۴")
|
|
|
|
# 5) create_ambiguities builds an entity_match entry with percentages
|
|
payload3 = {"items": [], "ambiguities": []}
|
|
jev_map = {normalize_text("منشیانه"): {
|
|
"field": "project", "status": "ambiguity", "variants": ["منشیانه"],
|
|
"candidates": [{"id": "p2", "name": "منشانه", "prob": 0.8},
|
|
{"id": "p1", "name": "دفتر", "prob": 0.02}]}}
|
|
matching.create_ambiguities(payload3, jev_map)
|
|
ambs = payload3["ambiguities"]
|
|
check("entity_match ambiguity created with candidates + details",
|
|
len(ambs) == 1 and ambs[0]["kind"] == "entity_match"
|
|
and len(ambs[0]["candidates"]) == 2 and "۸۰٪" in ambs[0]["details"])
|
|
matching.create_ambiguities(payload3, jev_map) # idempotent
|
|
check("create_ambiguities is deduped", len(payload3["ambiguities"]) == 1)
|
|
|
|
matching.jev_ask = orig_ask
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# live section (real NLU + real Jev) — opt-in and Bale-free
|
|
# ---------------------------------------------------------------------------
|
|
|
|
HARNESS_TRANSCRIPT = (
|
|
"مدل 1 (Whisper): الان شبت بخیر می خواستیم که بیست و پنج تا کیسه گچ و خاک از مصطفوی بخریم "
|
|
"و کنارش هم یه پنجاه تا کیسه سیمان پنجاه کیلویی. خریدمون هم عمدتا برای منشیانه و دفتر. "
|
|
"پیش فاکتورم گفته تا حدودا بیست روزی اوکیه.\n"
|
|
"مدل 2 (Vosk): میخواستیم 25 تا کیسه گچ و خاک از مستفدی بخریم و کنارشم یه 50 تا کیسه سیمان "
|
|
"50 کیلویی، خریدمون هم برای منشیانه و دفتر."
|
|
)
|
|
|
|
|
|
async def live_tests() -> None:
|
|
from app.api.client import ErpClient
|
|
from app.resolve import Resolver
|
|
from app.nlu import understand
|
|
|
|
api = ErpClient(settings.API_BASE_URL, settings.API_TIMEOUT_SECONDS, settings.SERVICE_TOKEN)
|
|
ctx = SimpleNamespace(resolver=Resolver(api))
|
|
|
|
print("\n--- LIVE: NLU entity_mentions ---")
|
|
result = await understand(settings, HARNESS_TRANSCRIPT)
|
|
mentions = result.get("entity_mentions") or []
|
|
print("intent:", result.get("intent"), "| mentions:", mentions)
|
|
check("live NLU produced entity_mentions", len(mentions) >= 2, f"{len(mentions)} mentions")
|
|
sup = next((m for m in mentions if m["field"] == "supplier"), None)
|
|
check("voice spellings grouped into ONE supplier mention",
|
|
bool(sup) and len(sup["variants"]) >= 2, str(sup))
|
|
|
|
print("\n--- LIVE: Jev per-field matching (real data) ---")
|
|
jev_map = await matching.resolve_mentions(ctx, result.get("patch") or {}, mentions, None)
|
|
for key, r in sorted(jev_map.items()):
|
|
print(f" {key!r}: {r['status']} -> {r.get('name')!r} conf={r.get('confidence')}")
|
|
check("live matching returned results", len(jev_map) > 0, f"{len(jev_map)} variants")
|
|
|
|
|
|
async def main() -> None:
|
|
await mock_tests()
|
|
if os.environ.get("RUN_LIVE") == "1":
|
|
if not settings.TYPESAFE_API_KEY:
|
|
check("live section requires TYPESAFE_API_KEY", False, "bot/.env not mounted?")
|
|
else:
|
|
await live_tests()
|
|
else:
|
|
print("\n(live section skipped — run with RUN_LIVE=1 to hit real NLU + Jev)")
|
|
|
|
total = len(PASSED) + len(FAILED)
|
|
print(f"\n{len(PASSED)}/{total} passed")
|
|
if FAILED:
|
|
print("FAILED:", FAILED)
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|