feat(bot): add DeepSeek crisis fallback engine for Jev failure and remove difflib heuristics
- Add app/fallback_matcher.py for intelligent crisis entity matching with token and latency observability - Route Jev failure/timeouts in app/matching.py to DeepSeek fallback engine - Handle MatchingFailureError defensively in handlers/__init__.py with honest error message - Enforce strict 'new' entity status in handlers/quote.py and implement defensive get_or_create on commit - Clean up app/resolve.py to eliminate difflib and character ratio heuristics - Pass all smoke tests (59/59), fallback matching tests (15/15), and matching unit tests (13/13)
This commit is contained in:
@@ -0,0 +1,309 @@
|
||||
"""Unit and integration tests for DeepSeek crisis fallback matching.
|
||||
|
||||
Covers:
|
||||
1. Jev failure -> jev.failure observability event + DeepSeek fallback invocation.
|
||||
2. DeepSeek fallback results mapping: exact, ambiguity, new (with inferred category).
|
||||
3. Defensive failure guard: DeepSeek network/HTTP/JSON failure raises MatchingFailureError.
|
||||
4. quote.py resolve_payload: respects new, binds exact, populates ambiguity, zero difflib.
|
||||
5. quote.py commit: defensive get_or_create pattern prevents 409 duplicate errors.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app import fallback_matcher, matching
|
||||
from app.config import settings
|
||||
from app.fallback_matcher import MatchingFailureError
|
||||
from app.handlers import quote
|
||||
from app.nlu import normalize_text
|
||||
from app.resolve import Candidate
|
||||
|
||||
PASSED: list[str] = []
|
||||
FAILED: list[str] = []
|
||||
|
||||
|
||||
def check(name: str, ok: bool, extra: str = "") -> None:
|
||||
(PASSED if ok else FAILED).append(name)
|
||||
print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}")
|
||||
|
||||
|
||||
class FakeResolver:
|
||||
async def projects(self):
|
||||
return [Candidate(id="p1", name="پروژه پردیس"), Candidate(id="p2", name="برج نگین")]
|
||||
|
||||
async def all_suppliers(self):
|
||||
return [Candidate(
|
||||
id="s1",
|
||||
name="فولاد خوزستان",
|
||||
extra={"location": "اهواز", "category_name": "فلزات و مقاطع فولادی", "slug": "metals-steel-sections"}
|
||||
)]
|
||||
|
||||
async def all_materials(self):
|
||||
return [
|
||||
Candidate(id="m1", name="میلگرد ۱۴ آجدار", extra={"canonical_unit_id": 1}),
|
||||
Candidate(id="m2", name="لوله پی‌وی‌سی ۴ اینچ", extra={"canonical_unit_id": 3}),
|
||||
]
|
||||
|
||||
async def categories(self):
|
||||
return [
|
||||
Candidate(id="cat1", name="فلزات و مقاطع فولادی", extra={"slug": "metals-steel-sections"}),
|
||||
Candidate(id="cat2", name="تأسیسات مکانیکی و آبی", extra={"slug": "mechanical-installations-pipes"}),
|
||||
]
|
||||
|
||||
async def units(self):
|
||||
return [Candidate(id="u1", name="شاخه", extra={"code": "branch"}), Candidate(id="u2", name="کیلوگرم", extra={"code": "kg"})]
|
||||
|
||||
def invalidate(self, key=None):
|
||||
pass
|
||||
|
||||
|
||||
class FakeApi:
|
||||
def __init__(self):
|
||||
self.created_materials = []
|
||||
self.created_suppliers = []
|
||||
self.created_projects = []
|
||||
|
||||
async def list_materials(self, q=None, page_size=20, page=1):
|
||||
if q and "میلگرد ۱۴" in q:
|
||||
return [{"id": "m1", "name": "میلگرد ۱۴ آجدار", "canonical_unit_id": 1}]
|
||||
return []
|
||||
|
||||
async def create_material(self, body, actor_id=None):
|
||||
if body["name"] == "میلگرد ۱۴ آجدار":
|
||||
raise Exception("Conflict 409")
|
||||
item = {"id": f"m_new_{len(self.created_materials)+1}", **body}
|
||||
self.created_materials.append(item)
|
||||
return item
|
||||
|
||||
async def list_suppliers(self, q=None, page_size=20, page=1):
|
||||
if q and "فولاد خوزستان" in q:
|
||||
return [{"id": "s1", "name": "فولاد خوزستان"}]
|
||||
return []
|
||||
|
||||
async def create_supplier(self, body, actor_id=None):
|
||||
item = {"id": f"s_new_{len(self.created_suppliers)+1}", **body}
|
||||
self.created_suppliers.append(item)
|
||||
return item
|
||||
|
||||
async def list_projects(self, page_size=100):
|
||||
return [{"id": "p1", "name": "پروژه پردیس"}]
|
||||
|
||||
async def create_project(self, body, actor_id=None):
|
||||
item = {"id": f"p_new_{len(self.created_projects)+1}", **body}
|
||||
self.created_projects.append(item)
|
||||
return item
|
||||
|
||||
|
||||
async def test_jev_failure_triggers_fallback() -> None:
|
||||
"""1. When Jev returns None (HTTP 500/timeout), DeepSeek fallback is triggered."""
|
||||
ctx = SimpleNamespace(resolver=FakeResolver())
|
||||
events_logged = []
|
||||
|
||||
async def fake_event(trace_id, event_type, ok=True, **kwargs):
|
||||
events_logged.append((event_type, kwargs))
|
||||
|
||||
orig_event = matching.obs.event
|
||||
matching.obs.event = fake_event
|
||||
|
||||
saved_key = settings.TYPESAFE_API_KEY
|
||||
settings.TYPESAFE_API_KEY = "test-typesafe-key"
|
||||
|
||||
orig_jev_ask = matching.jev_ask
|
||||
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result=None)
|
||||
|
||||
# Mock DeepSeek fallback matcher
|
||||
async def fake_fallback(raw_input, mentions, candidates_by_field, patch_context, trace_id):
|
||||
return {
|
||||
normalize_text("میلگرد ۱۴"): {
|
||||
"field": "material",
|
||||
"status": "exact",
|
||||
"id": "m1",
|
||||
"name": "میلگرد ۱۴ آجدار",
|
||||
"variants": ["میلگرد ۱۴"],
|
||||
"candidates": [],
|
||||
},
|
||||
normalize_text("لوله ۶ اینچ"): {
|
||||
"field": "material",
|
||||
"status": "new",
|
||||
"id": None,
|
||||
"name": None,
|
||||
"variants": ["لوله ۶ اینچ"],
|
||||
"candidates": [],
|
||||
},
|
||||
normalize_text("آهن"): {
|
||||
"field": "material",
|
||||
"status": "ambiguity",
|
||||
"id": None,
|
||||
"name": None,
|
||||
"variants": ["آهن"],
|
||||
"candidates": [{"id": "m1", "name": "میلگرد ۱۴ آجدار", "prob": 0.8}],
|
||||
},
|
||||
normalize_text("تأمین‌کننده البرز"): {
|
||||
"field": "supplier",
|
||||
"status": "new",
|
||||
"id": None,
|
||||
"name": None,
|
||||
"variants": ["تأمین‌کننده البرز"],
|
||||
"candidates": [],
|
||||
"inferred_category_slug": "metals-steel-sections",
|
||||
},
|
||||
}
|
||||
|
||||
orig_deepseek = matching.match_with_deepseek
|
||||
matching.match_with_deepseek = fake_fallback
|
||||
|
||||
try:
|
||||
mentions = [
|
||||
{"field": "material", "variants": ["میلگرد ۱۴"]},
|
||||
{"field": "material", "variants": ["لوله ۶ اینچ"]},
|
||||
{"field": "material", "variants": ["آهن"]},
|
||||
{"field": "supplier", "variants": ["تأمین‌کننده البرز"]},
|
||||
]
|
||||
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, trace_id="t1", raw_input="تست")
|
||||
|
||||
check("Jev failure recorded in observability", any(e[0] == "jev.failure" for e in events_logged))
|
||||
check("Exact match resolved via fallback", res[normalize_text("میلگرد ۱۴")]["status"] == "exact")
|
||||
check("New item strictly preserved as new (no difflib forced match)",
|
||||
res[normalize_text("لوله ۶ اینچ")]["status"] == "new" and res[normalize_text("لوله ۶ اینچ")]["id"] is None)
|
||||
check("Ambiguity item has candidates",
|
||||
res[normalize_text("آهن")]["status"] == "ambiguity" and len(res[normalize_text("آهن")]["candidates"]) == 1)
|
||||
check("New supplier has inferred_category_slug",
|
||||
res[normalize_text("تأمین‌کننده البرز")]["inferred_category_slug"] == "metals-steel-sections")
|
||||
|
||||
finally:
|
||||
settings.TYPESAFE_API_KEY = saved_key
|
||||
matching.obs.event = orig_event
|
||||
matching.jev_ask = orig_jev_ask
|
||||
matching.match_with_deepseek = orig_deepseek
|
||||
|
||||
|
||||
async def test_defensive_failure_guard() -> None:
|
||||
"""2. If DeepSeek fallback fails or returns unparseable JSON, MatchingFailureError is raised."""
|
||||
ctx = SimpleNamespace(resolver=FakeResolver())
|
||||
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result=None)
|
||||
|
||||
async def boom_fallback(raw_input, mentions, candidates_by_field, patch_context, trace_id):
|
||||
raise MatchingFailureError("DeepSeek API 500 error")
|
||||
|
||||
orig_deepseek = matching.match_with_deepseek
|
||||
matching.match_with_deepseek = boom_fallback
|
||||
|
||||
try:
|
||||
mentions = [{"field": "material", "variants": ["سیمان"]}]
|
||||
failed = False
|
||||
try:
|
||||
await matching.resolve_mentions(ctx, {"items": []}, mentions, trace_id="t2")
|
||||
except MatchingFailureError:
|
||||
failed = True
|
||||
check("Crisis fallback failure raises MatchingFailureError (stops cleanly)", failed)
|
||||
finally:
|
||||
matching.match_with_deepseek = orig_deepseek
|
||||
|
||||
|
||||
async def test_quote_resolve_payload_zero_difflib() -> None:
|
||||
"""3. resolve_payload in quote.py respects Jev/fallback map and never runs difflib."""
|
||||
ctx = SimpleNamespace(resolver=FakeResolver())
|
||||
payload = {
|
||||
"supplier_name": "تأمین‌کننده البرز",
|
||||
"items": [
|
||||
{"material_name": "لوله ۶ اینچ", "unit_name": "شاخه"},
|
||||
{"material_name": "میلگرد ۱۴", "unit_name": "شاخه"},
|
||||
],
|
||||
}
|
||||
jev_map = {
|
||||
normalize_text("تأمین‌کننده البرز"): {
|
||||
"field": "supplier", "status": "new", "variants": ["تأمین‌کننده البرز"],
|
||||
"inferred_category_slug": "metals-steel-sections",
|
||||
},
|
||||
normalize_text("لوله ۶ اینچ"): {
|
||||
"field": "material", "status": "new", "variants": ["لوله ۶ اینچ"],
|
||||
},
|
||||
normalize_text("میلگرد ۱۴"): {
|
||||
"field": "material", "status": "exact", "id": "m1", "name": "میلگرد ۱۴ آجدار",
|
||||
"variants": ["میلگرد ۱۴"],
|
||||
},
|
||||
normalize_text("شاخه"): {
|
||||
"field": "unit", "status": "exact", "id": "u1", "name": "شاخه", "variants": ["شاخه"],
|
||||
},
|
||||
}
|
||||
|
||||
await quote.resolve_payload(ctx, payload, trace_id="t3", jev_map=jev_map)
|
||||
|
||||
check("New supplier kept id=None (no difflib forced match)", payload.get("supplier_id") is None)
|
||||
check("New supplier inferred category saved", payload.get("supplier_inferred_category_slug") == "metals-steel-sections")
|
||||
check("New material kept id=None (no difflib forced match to 4-inch pipe)", payload["items"][0].get("material_id") is None)
|
||||
check("Exact material bound to catalog ID", payload["items"][1].get("material_id") == "m1")
|
||||
check("Exact unit bound to catalog ID", payload["items"][0].get("unit_id") == "u1")
|
||||
|
||||
|
||||
async def test_quote_commit_defensive_get_or_create() -> None:
|
||||
"""4. commit in quote.py uses get_or_create to prevent 409 conflict errors."""
|
||||
api = FakeApi()
|
||||
resolver = FakeResolver()
|
||||
ctx = SimpleNamespace(api=api, resolver=resolver, pending={},
|
||||
send=lambda *a, **k: asyncio.sleep(0.0001))
|
||||
user = SimpleNamespace(actor_id="usr_1")
|
||||
|
||||
# Mock obs calls so offline test doesn't require live DB connection
|
||||
orig_latest = quote.obs.latest_trace_for_draft
|
||||
orig_link = quote.obs.link_draft
|
||||
orig_event = quote.obs.event
|
||||
quote.obs.latest_trace_for_draft = lambda did: asyncio.sleep(0.0001, result="mock_trace")
|
||||
quote.obs.link_draft = lambda *a, **k: asyncio.sleep(0.0001)
|
||||
quote.obs.event = lambda *a, **k: asyncio.sleep(0.0001)
|
||||
|
||||
# Item with name "میلگرد ۱۴ آجدار" that already exists in DB
|
||||
payload = {
|
||||
"supplier_name": "فولاد خوزستان", # Already in DB
|
||||
"supplier_inferred_category_slug": "metals-steel-sections",
|
||||
"items": [
|
||||
{"material_name": "میلگرد ۱۴ آجدار", "unit_id": 1, "quantity": 10, "unit_price": 25000},
|
||||
{"material_name": "تیرآهن ۱۸", "unit_id": 1, "quantity": 5, "unit_price": 50000}, # Brand new
|
||||
],
|
||||
"project_name": "پروژه پردیس", # Already in DB
|
||||
}
|
||||
draft = SimpleNamespace(id="d1", payload=payload, review_message_id=None)
|
||||
|
||||
# Mock create_quote
|
||||
api.create_quote = lambda body, actor_id: asyncio.sleep(0.001, result={"id": "q1"})
|
||||
# Mock draft_store.set_status
|
||||
quote.draft_store.set_status = lambda did, st: asyncio.sleep(0.001)
|
||||
|
||||
try:
|
||||
await quote.commit(ctx, chat_id=123, user=user, draft=draft, only_resolved=False)
|
||||
|
||||
check("Existing supplier reused ID from DB (not recreated)", payload["supplier_id"] == "s1")
|
||||
check("Existing material reused ID from DB on conflict (not crashing)", payload["items"][0]["material_id"] == "m1")
|
||||
check("New material created cleanly", payload["items"][1]["material_id"] == "m_new_1")
|
||||
check("Existing project reused ID from DB", payload["project_id"] == "p1")
|
||||
finally:
|
||||
quote.obs.latest_trace_for_draft = orig_latest
|
||||
quote.obs.link_draft = orig_link
|
||||
quote.obs.event = orig_event
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print("Running DeepSeek Crisis Fallback & Zero-Difflib Tests...\n")
|
||||
orig_fb_event = fallback_matcher.obs.event
|
||||
fallback_matcher.obs.event = lambda *a, **k: asyncio.sleep(0.0001)
|
||||
try:
|
||||
await test_jev_failure_triggers_fallback()
|
||||
await test_defensive_failure_guard()
|
||||
await test_quote_resolve_payload_zero_difflib()
|
||||
await test_quote_commit_defensive_get_or_create()
|
||||
finally:
|
||||
fallback_matcher.obs.event = orig_fb_event
|
||||
|
||||
total = len(PASSED) + len(FAILED)
|
||||
print(f"\n{len(PASSED)}/{total} passed")
|
||||
if FAILED:
|
||||
print("FAILED:", FAILED)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -74,13 +74,24 @@ async def mock_tests() -> None:
|
||||
"mention_3": {"choice": "دفتر", "confidence": 0.5,
|
||||
"probabilities": {"دفتر": 0.52, "_none": 0.48}},
|
||||
})
|
||||
orig_fallback = matching.match_with_deepseek
|
||||
async def mock_fallback_below_floor(*args, **kwargs):
|
||||
ms = kwargs.get("mentions") or (args[1] if len(args) > 1 else [])
|
||||
return {normalize_text(v): {"status": "new", "field": m.get("field"), "variants": m.get("variants"), "id": None}
|
||||
for m in ms for v in m.get("variants", [])}
|
||||
matching.match_with_deepseek = mock_fallback_below_floor
|
||||
|
||||
mentions = [
|
||||
{"field": "project", "variants": ["منشیانه"]},
|
||||
{"field": "project", "variants": ["دفتر", "دفر"]},
|
||||
{"field": "project", "variants": ["چیز عجیب"]},
|
||||
{"field": "project", "variants": ["دفر دو"]},
|
||||
]
|
||||
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
try:
|
||||
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
finally:
|
||||
matching.match_with_deepseek = orig_fallback
|
||||
|
||||
r0 = matching.lookup(res, "منشیانه")
|
||||
r1 = matching.lookup(res, "دفر") # variant key resolves too
|
||||
r2 = matching.lookup(res, "چیز عجیب")
|
||||
@@ -90,15 +101,23 @@ async def mock_tests() -> None:
|
||||
check("variant of same mention shares one result", r1 is r0 if False else
|
||||
(r1 and matching.lookup(res, "دفتر") is r1))
|
||||
check("_none / low conf -> new", r2 and r2["status"] == "new")
|
||||
check("below-floor pick -> NO result (difflib fallback keeps its chance)",
|
||||
matching.lookup(res, "دفر دو") is None)
|
||||
check("below-floor pick -> routed to crisis fallback",
|
||||
matching.lookup(res, "دفر دو") and matching.lookup(res, "دفر دو")["status"] == "new")
|
||||
check("below-floor pick did not pollute other mentions", r1 and r1["status"] == "ambiguity")
|
||||
|
||||
# 2) Jev failure -> empty map (callers fall back to difflib)
|
||||
# 2) Jev failure -> DeepSeek crisis fallback
|
||||
matching.jev_ask = make_fake_ask({})
|
||||
matching.jev_ask.fail = True
|
||||
res_fail = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
check("jev failure -> empty results (difflib fallback path)", res_fail == {})
|
||||
async def mock_deepseek(*args, **kwargs):
|
||||
return {normalize_text("منشیانه"): {"status": "exact", "id": "p2", "name": "منشانه",
|
||||
"field": "project", "variants": ["منشیانه"]}}
|
||||
matching.match_with_deepseek = mock_deepseek
|
||||
try:
|
||||
res_fail = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
check("jev failure -> triggers crisis fallback",
|
||||
normalize_text("منشیانه") in res_fail and res_fail[normalize_text("منشیانه")]["status"] == "exact")
|
||||
finally:
|
||||
matching.match_with_deepseek = orig_fallback
|
||||
|
||||
# 3) apply_exact writes supplier/project ids into a payload
|
||||
payload = {"items": []}
|
||||
|
||||
Reference in New Issue
Block a user