feat(matching): Jev entity matching phases 2-5 (per-field fan-out, variants, candidate ambiguity buttons)
Phase 2 — jev.py client: POST /v1/systemone, PINNED model jev-1.13.0 (drift warning when the response model differs), one retry on 429/5xx, fail-soft (None on any error), usage/latency returned for observability. Config adds TYPESAFE_API_KEY / JEV_* (thresholds 0.85/0.60, 15 questions per field); no daily cap by owner decision. bot/.env is now mounted read-only into the container so the key never enters compose/git.
Phase 3 — matching.py: برونگروهی (one request per field, parallel), درونگروهی (one choice question per mention sharing the same full candidate list + «_none»), درونواریانسی (all STT spellings in one question). Routing: conf>=exact → bind; floor..exact → ambiguity; else/_none → new; missing/failed field → difflib fallback (bug fixed during testing: a failed request used to record 'new' instead of falling back). Resolver gains full cached lists for suppliers/materials (paged) plus brands (API+DB) and specs (EAV values + draft specs). NLU prompt now emits entity_mentions (6 fields) with the voice-variant grouping rule; sanitize_mentions guards the shape. handle_message runs matching only for register_quote/update_draft/add_material behind JEV_ENABLED; jev_map threads into quote/material resolve_payload (Jev hit skips difflib; exact writes ids; ambiguity skips difflib). Spec effect per owner: spelling normalization only (normalize_brand_spec).
Phase 4 — entity-match ambiguity sub-messages: create_ambiguities() builds deduped entries {kind: entity_match, field, variants, candidates, details with Persian percentages}; new keyboards.ambiguity_candidates_keyboard renders one button per candidate + «هیچکدام — ایجاد جدید»; callbacks handle amb⛏️{prefix}:{amb_id}:{idx} (applies the candidate via matching.apply_choice then re-resolves/renders) and amb:new (rejected → auto-create path at commit). Every user choice lands in observability (ambiguity.user_choice with arg+candidates).
Phase 5 — voice variants: NLU groups different STT spellings of one entity into one variants[] (verified live: مصطفوی/مستفدی → one supplier mention); tools/test_matching.py live section (RUN_LIVE=1) exercises real DeepSeek + real Jev without touching Bale.
Tests: test_jev.py 5/5 (mock HTTP: no-key, happy path, 429-retry, 400, exception), test_matching.py 11/11 mock + 3/3 live (real NLU produced 7 mentions with correct fields; real Jev matched «50 کیلویی»→«۵۰ کیلویی» 0.98 and «کیسه» 1.0 and correctly returned new against the emptied catalog); observability regression 8/8.
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
"""Phase-2 tests for the Jev client — MOCKED HTTP (no network, no key needed).
|
||||
|
||||
docker compose exec bot python tools/test_jev.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app import jev
|
||||
from app.config import settings
|
||||
|
||||
PASSED: list[str] = []
|
||||
FAILED: list[str] = []
|
||||
|
||||
|
||||
def check(name: str, ok: bool, extra: str = "") -> None:
|
||||
(PASSED if ok else FAILED).append(name)
|
||||
print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}")
|
||||
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, status: int, body: dict):
|
||||
self.status_code = status
|
||||
self._body = body
|
||||
self.text = str(body)
|
||||
|
||||
def json(self):
|
||||
return self._body
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""Scripted httpx.AsyncClient stand-in: pops one (status, body) per POST."""
|
||||
|
||||
script: list = []
|
||||
posts: int = 0
|
||||
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def post(self, url, json=None, headers=None):
|
||||
FakeClient.posts += 1
|
||||
status, body = FakeClient.script.pop(0)
|
||||
return FakeResp(status, body)
|
||||
|
||||
|
||||
Q = {"m0": {"type": "choice", "instructions": "i", "criteria": {"x": ""}}}
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
orig_client = jev.httpx.AsyncClient
|
||||
saved_key = settings.TYPESAFE_API_KEY
|
||||
jev.httpx.AsyncClient = FakeClient
|
||||
try:
|
||||
# 1) no key -> None (no HTTP at all)
|
||||
settings.TYPESAFE_API_KEY = ""
|
||||
FakeClient.posts = 0
|
||||
r = await jev.ask({"s": 1}, Q)
|
||||
check("no api key -> None (no request)", r is None and FakeClient.posts == 0)
|
||||
|
||||
settings.TYPESAFE_API_KEY = saved_key or "test-key"
|
||||
|
||||
# 2) happy path: answers + model + usage + latency
|
||||
FakeClient.script = [(200, {"model": "jev-1.13.0",
|
||||
"answers": {"m0": {"choice": "x", "confidence": 0.9,
|
||||
"probabilities": {"x": 0.9}}},
|
||||
"usage": {"input_tokens": 10, "output_tokens": 2}})]
|
||||
r = await jev.ask({"s": 1}, Q)
|
||||
ok = bool(r) and r["answers"]["m0"]["choice"] == "x" and r["model"] == "jev-1.13.0" \
|
||||
and r["usage"]["input_tokens"] == 10 and isinstance(r["latency_ms"], int)
|
||||
check("happy path -> answers/model/usage/latency", ok)
|
||||
|
||||
# 3) 429 -> one retry, then success
|
||||
FakeClient.script = [(429, {"error": "rate_limited"}),
|
||||
(200, {"model": "jev-1.13.0", "answers": {}, "usage": {}})]
|
||||
FakeClient.posts = 0
|
||||
r = await jev.ask({"s": 1}, Q)
|
||||
check("429 retried once then succeeds", r is not None and FakeClient.posts == 2)
|
||||
|
||||
# 4) hard 400 -> None, no retry
|
||||
FakeClient.script = [(400, {"error": "bad_request"})]
|
||||
FakeClient.posts = 0
|
||||
r = await jev.ask({"s": 1}, Q)
|
||||
check("400 -> None without retry", r is None and FakeClient.posts == 1)
|
||||
|
||||
# 5) exception -> retried once -> None
|
||||
class BoomClient(FakeClient):
|
||||
async def post(self, url, json=None, headers=None):
|
||||
FakeClient.posts += 1
|
||||
raise RuntimeError("net down")
|
||||
|
||||
jev.httpx.AsyncClient = BoomClient
|
||||
FakeClient.posts = 0
|
||||
r = await jev.ask({"s": 1}, Q)
|
||||
check("exception retried once -> None", r is None and FakeClient.posts == 2)
|
||||
finally:
|
||||
jev.httpx.AsyncClient = orig_client
|
||||
settings.TYPESAFE_API_KEY = saved_key
|
||||
|
||||
total = len(PASSED) + len(FAILED)
|
||||
print(f"\n{len(PASSED)}/{total} passed")
|
||||
if FAILED:
|
||||
print("FAILED:", FAILED)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Phase 3-5 tests for entity matching (mock by default; live with RUN_LIVE=1).
|
||||
|
||||
docker compose exec bot python tools/test_matching.py # mock only
|
||||
docker compose exec -e RUN_LIVE=1 bot python tools/test_matching.py # + real NLU + Jev
|
||||
|
||||
The live section calls DeepSeek (NLU) and TypeSafe (Jev) with the REAL keys
|
||||
from bot/.env — it never sends anything to Bale and never creates drafts.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app import matching
|
||||
from app.config import settings
|
||||
from app.nlu import normalize_text
|
||||
from app.resolve import Candidate
|
||||
|
||||
PASSED: list[str] = []
|
||||
FAILED: list[str] = []
|
||||
|
||||
|
||||
def check(name: str, ok: bool, extra: str = "") -> None:
|
||||
(PASSED if ok else FAILED).append(name)
|
||||
print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}")
|
||||
|
||||
|
||||
class FakeResolver:
|
||||
async def projects(self):
|
||||
return [Candidate(id="p1", name="دفتر"), Candidate(id="p2", name="منشانه")]
|
||||
|
||||
async def all_suppliers(self):
|
||||
return [Candidate(id="s1", name="بازرگانی مصطفوی",
|
||||
extra={"location": "اصفهان", "category_name": "مصالح"})]
|
||||
|
||||
async def all_materials(self):
|
||||
return [Candidate(id="m1", name="گچ و خاک", extra={"canonical_unit_id": 2})]
|
||||
|
||||
async def brands(self):
|
||||
return [Candidate(id="b1", name="سیمان تهران")]
|
||||
|
||||
async def specs(self):
|
||||
return [Candidate(id="سایز ۱۴", name="سایز ۱۴")]
|
||||
|
||||
async def units(self):
|
||||
return [Candidate(id="u1", name="کیسه"), Candidate(id="u2", name="عدد")]
|
||||
|
||||
|
||||
def make_fake_ask(answers_by_mention: dict, model: str = "jev-1.13.0"):
|
||||
async def fake_ask(state, questions):
|
||||
if fake_ask.fail:
|
||||
return None
|
||||
return {"answers": answers_by_mention, "usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
"model": model, "latency_ms": 7}
|
||||
fake_ask.fail = False
|
||||
return fake_ask
|
||||
|
||||
|
||||
async def mock_tests() -> None:
|
||||
ctx = SimpleNamespace(resolver=FakeResolver())
|
||||
orig_ask = matching.jev_ask
|
||||
|
||||
# 1) routing bands: exact / ambiguity / new + variant grouping
|
||||
matching.jev_ask = make_fake_ask({
|
||||
"mention_0": {"choice": "منشانه", "confidence": 0.9,
|
||||
"probabilities": {"منشانه": 0.95, "_none": 0.05}},
|
||||
"mention_1": {"choice": "دفتر", "confidence": 0.7,
|
||||
"probabilities": {"دفتر": 0.73, "منشانه": 0.15, "_none": 0.12}},
|
||||
"mention_2": {"choice": "_none", "confidence": 0.9,
|
||||
"probabilities": {"_none": 0.94}},
|
||||
})
|
||||
mentions = [
|
||||
{"field": "project", "variants": ["منشیانه"]},
|
||||
{"field": "project", "variants": ["دفتر", "دفر"]},
|
||||
{"field": "project", "variants": ["چیز عجیب"]},
|
||||
]
|
||||
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
r0 = matching.lookup(res, "منشیانه")
|
||||
r1 = matching.lookup(res, "دفر") # variant key resolves too
|
||||
r2 = matching.lookup(res, "چیز عجیب")
|
||||
check("exact band -> status exact + id", r0 and r0["status"] == "exact" and r0["id"] == "p2")
|
||||
check("mid band -> ambiguity + candidates", r1 and r1["status"] == "ambiguity"
|
||||
and r1["candidates"][0]["name"] == "دفتر")
|
||||
check("variant of same mention shares one result", r1 is r0 if False else
|
||||
(r1 and matching.lookup(res, "دفتر") is r1))
|
||||
check("_none / low conf -> new", r2 and r2["status"] == "new")
|
||||
|
||||
# 2) Jev failure -> empty map (callers fall back to difflib)
|
||||
matching.jev_ask = make_fake_ask({})
|
||||
matching.jev_ask.fail = True
|
||||
res_fail = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
check("jev failure -> empty results (difflib fallback path)", res_fail == {})
|
||||
|
||||
# 3) apply_exact writes supplier/project ids into a payload
|
||||
payload = {"items": []}
|
||||
matching.apply_exact(payload, {"field": "project", "id": "p2", "name": "منشانه",
|
||||
"variants": ["منشیانه"], "scores": {}})
|
||||
matching.apply_exact(payload, {"field": "supplier", "id": "s1", "name": "بازرگانی مصطفوی",
|
||||
"extra": {"location": "اصفهان"}, "variants": ["مستفدی"]})
|
||||
check("apply_exact -> project_id/resolved_projects/project_ids",
|
||||
payload.get("project_id") == "p2"
|
||||
and payload["resolved_projects"].get("منشیانه") == "p2")
|
||||
check("apply_exact -> supplier_id/name/extra",
|
||||
payload.get("supplier_id") == "s1" and payload.get("supplier_location") == "اصفهان")
|
||||
|
||||
# 4) material/unit/brand/spec application on items
|
||||
payload2 = {"items": [{"material_name": "مصطفی", "unit_name": "کیسه",
|
||||
"brand_name": "سیمان تهران", "specs": ["سایز 14"]}]}
|
||||
matching.apply_choice(payload2, "material", {"id": "m1", "name": "گچ و خاک",
|
||||
"extra": {"canonical_unit_id": 2}},
|
||||
["مصطفی"])
|
||||
matching.apply_choice(payload2, "unit", {"id": "u1", "name": "کیسه"}, ["کیسه"])
|
||||
matching.normalize_brand_spec(payload2, {normalize_text("سایز 14"): {
|
||||
"field": "spec", "status": "exact", "name": "سایز ۱۴", "id": "سایز ۱۴",
|
||||
"variants": ["سایز 14"]}})
|
||||
it = payload2["items"][0]
|
||||
check("material/unit applied on matching items",
|
||||
it["material_id"] == "m1" and it["unit_id"] == "u1" and it["unit_explicit"] is True)
|
||||
check("spec normalization (spelling only)", it["specs"][0] == "سایز ۱۴")
|
||||
|
||||
# 5) create_ambiguities builds an entity_match entry with percentages
|
||||
payload3 = {"items": [], "ambiguities": []}
|
||||
jev_map = {normalize_text("منشیانه"): {
|
||||
"field": "project", "status": "ambiguity", "variants": ["منشیانه"],
|
||||
"candidates": [{"id": "p2", "name": "منشانه", "prob": 0.8},
|
||||
{"id": "p1", "name": "دفتر", "prob": 0.02}]}}
|
||||
matching.create_ambiguities(payload3, jev_map)
|
||||
ambs = payload3["ambiguities"]
|
||||
check("entity_match ambiguity created with candidates + details",
|
||||
len(ambs) == 1 and ambs[0]["kind"] == "entity_match"
|
||||
and len(ambs[0]["candidates"]) == 2 and "۸۰٪" in ambs[0]["details"])
|
||||
matching.create_ambiguities(payload3, jev_map) # idempotent
|
||||
check("create_ambiguities is deduped", len(payload3["ambiguities"]) == 1)
|
||||
|
||||
matching.jev_ask = orig_ask
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# live section (real NLU + real Jev) — opt-in and Bale-free
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HARNESS_TRANSCRIPT = (
|
||||
"مدل 1 (Whisper): الان شبت بخیر می خواستیم که بیست و پنج تا کیسه گچ و خاک از مصطفوی بخریم "
|
||||
"و کنارش هم یه پنجاه تا کیسه سیمان پنجاه کیلویی. خریدمون هم عمدتا برای منشیانه و دفتر. "
|
||||
"پیش فاکتورم گفته تا حدودا بیست روزی اوکیه.\n"
|
||||
"مدل 2 (Vosk): میخواستیم 25 تا کیسه گچ و خاک از مستفدی بخریم و کنارشم یه 50 تا کیسه سیمان "
|
||||
"50 کیلویی، خریدمون هم برای منشیانه و دفتر."
|
||||
)
|
||||
|
||||
|
||||
async def live_tests() -> None:
|
||||
from app.api.client import ErpClient
|
||||
from app.resolve import Resolver
|
||||
from app.nlu import understand
|
||||
|
||||
api = ErpClient(settings.API_BASE_URL, settings.API_TIMEOUT_SECONDS, settings.SERVICE_TOKEN)
|
||||
ctx = SimpleNamespace(resolver=Resolver(api))
|
||||
|
||||
print("\n--- LIVE: NLU entity_mentions ---")
|
||||
result = await understand(settings, HARNESS_TRANSCRIPT)
|
||||
mentions = result.get("entity_mentions") or []
|
||||
print("intent:", result.get("intent"), "| mentions:", mentions)
|
||||
check("live NLU produced entity_mentions", len(mentions) >= 2, f"{len(mentions)} mentions")
|
||||
sup = next((m for m in mentions if m["field"] == "supplier"), None)
|
||||
check("voice spellings grouped into ONE supplier mention",
|
||||
bool(sup) and len(sup["variants"]) >= 2, str(sup))
|
||||
|
||||
print("\n--- LIVE: Jev per-field matching (real data) ---")
|
||||
jev_map = await matching.resolve_mentions(ctx, result.get("patch") or {}, mentions, None)
|
||||
for key, r in sorted(jev_map.items()):
|
||||
print(f" {key!r}: {r['status']} -> {r.get('name')!r} conf={r.get('confidence')}")
|
||||
check("live matching returned results", len(jev_map) > 0, f"{len(jev_map)} variants")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
await mock_tests()
|
||||
if os.environ.get("RUN_LIVE") == "1":
|
||||
if not settings.TYPESAFE_API_KEY:
|
||||
check("live section requires TYPESAFE_API_KEY", False, "bot/.env not mounted?")
|
||||
else:
|
||||
await live_tests()
|
||||
else:
|
||||
print("\n(live section skipped — run with RUN_LIVE=1 to hit real NLU + Jev)")
|
||||
|
||||
total = len(PASSED) + len(FAILED)
|
||||
print(f"\n{len(PASSED)}/{total} passed")
|
||||
if FAILED:
|
||||
print("FAILED:", FAILED)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user