404 lines
17 KiB
Python
404 lines
17 KiB
Python
"""Unit and integration tests for DeepSeek crisis fallback matching.
|
|
|
|
Covers:
|
|
1. Jev failure -> jev.failure observability event + DeepSeek fallback invocation.
|
|
2. DeepSeek fallback results mapping: exact, ambiguity, new (with inferred category).
|
|
3. Defensive failure guard: DeepSeek network/HTTP/JSON failure raises MatchingFailureError.
|
|
4. quote.py resolve_payload: respects new, binds exact, populates ambiguity, zero difflib.
|
|
5. quote.py commit: defensive get_or_create pattern prevents 409 duplicate errors.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app import fallback_matcher, matching
|
|
from app.config import settings
|
|
from app.fallback_matcher import MatchingFailureError
|
|
from app.handlers import quote
|
|
from app.nlu import normalize_text
|
|
from app.resolve import Candidate
|
|
|
|
PASSED: list[str] = []
|
|
FAILED: list[str] = []
|
|
|
|
|
|
def check(name: str, ok: bool, extra: str = "") -> None:
|
|
(PASSED if ok else FAILED).append(name)
|
|
print(f"{'PASS' if ok else 'FAIL'} {name}{' — ' + str(extra) if extra else ''}")
|
|
|
|
|
|
class FakeResolver:
|
|
async def projects(self):
|
|
return [Candidate(id="p1", name="پروژه پردیس"), Candidate(id="p2", name="برج نگین")]
|
|
|
|
async def all_suppliers(self):
|
|
return [Candidate(
|
|
id="s1",
|
|
name="فولاد خوزستان",
|
|
extra={"location": "اهواز", "category_name": "فلزات و مقاطع فولادی", "slug": "metals-steel-sections"}
|
|
)]
|
|
|
|
async def all_materials(self):
|
|
return [
|
|
Candidate(id="m1", name="میلگرد ۱۴ آجدار", extra={"canonical_unit_id": 1}),
|
|
Candidate(id="m2", name="لوله پی‌وی‌سی ۴ اینچ", extra={"canonical_unit_id": 3}),
|
|
]
|
|
|
|
async def categories(self):
|
|
return [
|
|
Candidate(id="cat1", name="فلزات و مقاطع فولادی", extra={"slug": "metals-steel-sections"}),
|
|
Candidate(id="cat2", name="تأسیسات مکانیکی و آبی", extra={"slug": "mechanical-installations-pipes"}),
|
|
]
|
|
|
|
async def units(self):
|
|
return [Candidate(id="u1", name="شاخه", extra={"code": "branch"}), Candidate(id="u2", name="کیلوگرم", extra={"code": "kg"})]
|
|
|
|
def invalidate(self, key=None):
|
|
pass
|
|
|
|
|
|
class FakeApi:
|
|
def __init__(self):
|
|
self.created_materials = []
|
|
self.created_suppliers = []
|
|
self.created_projects = []
|
|
|
|
async def list_materials(self, q=None, page_size=20, page=1):
|
|
if q and "میلگرد ۱۴" in q:
|
|
return [{"id": "m1", "name": "میلگرد ۱۴ آجدار", "canonical_unit_id": 1}]
|
|
return []
|
|
|
|
async def create_material(self, body, actor_id=None):
|
|
if body["name"] == "میلگرد ۱۴ آجدار":
|
|
raise Exception("Conflict 409")
|
|
item = {"id": f"m_new_{len(self.created_materials)+1}", **body}
|
|
self.created_materials.append(item)
|
|
return item
|
|
|
|
async def list_suppliers(self, q=None, page_size=20, page=1):
|
|
if q and "فولاد خوزستان" in q:
|
|
return [{"id": "s1", "name": "فولاد خوزستان"}]
|
|
return []
|
|
|
|
async def create_supplier(self, body, actor_id=None):
|
|
item = {"id": f"s_new_{len(self.created_suppliers)+1}", **body}
|
|
self.created_suppliers.append(item)
|
|
return item
|
|
|
|
async def list_projects(self, page_size=100):
|
|
return [{"id": "p1", "name": "پروژه پردیس"}]
|
|
|
|
async def create_project(self, body, actor_id=None):
|
|
item = {"id": f"p_new_{len(self.created_projects)+1}", **body}
|
|
self.created_projects.append(item)
|
|
return item
|
|
|
|
|
|
async def test_jev_failure_triggers_fallback() -> None:
|
|
"""1. When Jev returns None (HTTP 500/timeout), DeepSeek fallback is triggered."""
|
|
ctx = SimpleNamespace(resolver=FakeResolver())
|
|
events_logged = []
|
|
|
|
async def fake_event(trace_id, event_type, ok=True, **kwargs):
|
|
events_logged.append((event_type, kwargs))
|
|
|
|
orig_event = matching.obs.event
|
|
matching.obs.event = fake_event
|
|
|
|
saved_key = settings.TYPESAFE_API_KEY
|
|
settings.TYPESAFE_API_KEY = "test-typesafe-key"
|
|
|
|
orig_jev_ask = matching.jev_ask
|
|
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result=None)
|
|
|
|
# Mock DeepSeek fallback matcher
|
|
async def fake_fallback(*args, **kwargs):
|
|
return {
|
|
normalize_text("میلگرد ۱۴"): {
|
|
"field": "material",
|
|
"status": "exact",
|
|
"id": "m1",
|
|
"name": "میلگرد ۱۴ آجدار",
|
|
"variants": ["میلگرد ۱۴"],
|
|
"candidates": [],
|
|
},
|
|
normalize_text("لوله ۶ اینچ"): {
|
|
"field": "material",
|
|
"status": "new",
|
|
"id": None,
|
|
"name": None,
|
|
"variants": ["لوله ۶ اینچ"],
|
|
"candidates": [],
|
|
},
|
|
normalize_text("آهن"): {
|
|
"field": "material",
|
|
"status": "ambiguity",
|
|
"id": None,
|
|
"name": None,
|
|
"variants": ["آهن"],
|
|
"candidates": [{"id": "m1", "name": "میلگرد ۱۴ آجدار", "prob": 0.8}],
|
|
},
|
|
normalize_text("تأمین‌کننده البرز"): {
|
|
"field": "supplier",
|
|
"status": "new",
|
|
"id": None,
|
|
"name": None,
|
|
"variants": ["تأمین‌کننده البرز"],
|
|
"candidates": [],
|
|
"inferred_category_slug": "metals-steel-sections",
|
|
},
|
|
}
|
|
|
|
orig_deepseek = matching.match_with_deepseek
|
|
matching.match_with_deepseek = fake_fallback
|
|
|
|
try:
|
|
mentions = [
|
|
{"field": "material", "variants": ["میلگرد ۱۴"]},
|
|
{"field": "material", "variants": ["لوله ۶ اینچ"]},
|
|
{"field": "material", "variants": ["آهن"]},
|
|
{"field": "supplier", "variants": ["تأمین‌کننده البرز"]},
|
|
]
|
|
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, trace_id="t1", raw_input="تست")
|
|
|
|
check("Jev failure recorded in observability", any(e[0] == "jev.failure" for e in events_logged))
|
|
check("Exact match resolved via fallback", res[normalize_text("میلگرد ۱۴")]["status"] == "exact")
|
|
check("New item strictly preserved as new (no difflib forced match)",
|
|
res[normalize_text("لوله ۶ اینچ")]["status"] == "new" and res[normalize_text("لوله ۶ اینچ")]["id"] is None)
|
|
check("Ambiguity item has candidates",
|
|
res[normalize_text("آهن")]["status"] == "ambiguity" and len(res[normalize_text("آهن")]["candidates"]) == 1)
|
|
check("New supplier has inferred_category_slug",
|
|
res[normalize_text("تأمین‌کننده البرز")]["inferred_category_slug"] == "metals-steel-sections")
|
|
|
|
finally:
|
|
settings.TYPESAFE_API_KEY = saved_key
|
|
matching.obs.event = orig_event
|
|
matching.jev_ask = orig_jev_ask
|
|
matching.match_with_deepseek = orig_deepseek
|
|
|
|
|
|
async def test_two_stage_adjudication_flow() -> None:
|
|
"""2. When Jev succeeds, outputs are NOT directly committed; they are sent to DeepSeek Arbiter."""
|
|
ctx = SimpleNamespace(resolver=FakeResolver())
|
|
saved_key = settings.TYPESAFE_API_KEY
|
|
settings.TYPESAFE_API_KEY = "test-typesafe-key"
|
|
|
|
orig_jev_ask = matching.jev_ask
|
|
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result={
|
|
"answers": {
|
|
"mention_0": {
|
|
"choice": "میلگرد ۱۴ آجدار",
|
|
"confidence": 0.95,
|
|
"probabilities": {"میلگرد ۱۴ آجدار": 0.95, "تسمه فلزی": 0.0, "_none": 0.05},
|
|
},
|
|
"mention_1": {
|
|
"choice": "لوله پی‌وی‌سی ۴ اینچ",
|
|
"confidence": 0.82,
|
|
"probabilities": {"لوله پی‌وی‌سی ۴ اینچ": 0.82, "لوله پلی‌اتیلن": 0.0, "_none": 0.18},
|
|
},
|
|
}
|
|
})
|
|
|
|
arbiter_called_with = []
|
|
|
|
async def fake_arbiter(*args, **kwargs):
|
|
arbiter_called_with.append((args, kwargs))
|
|
items = kwargs.get("items") or (args[2] if len(args) > 2 else [])
|
|
res = {}
|
|
for it in items:
|
|
v0 = it["variants"][0]
|
|
if v0 == "میلگرد ۱۴":
|
|
# Endorsed
|
|
res[normalize_text(v0)] = {
|
|
"field": it["field"], "status": "exact", "id": "m1", "name": "میلگرد ۱۴ آجدار",
|
|
"variants": it["variants"], "source": "deepseek-arbiter", "adjudication_action": "endorsed_jev",
|
|
}
|
|
elif v0 == "لوله ۶ اینچ":
|
|
# Vetoed Jev (Jev wrongly picked 4-inch pipe)
|
|
res[normalize_text(v0)] = {
|
|
"field": it["field"], "status": "new", "id": None, "name": None,
|
|
"variants": it["variants"], "source": "deepseek-arbiter", "adjudication_action": "vetoed_to_new",
|
|
}
|
|
return res
|
|
|
|
orig_arbiter = matching.adjudicate_with_deepseek
|
|
matching.adjudicate_with_deepseek = fake_arbiter
|
|
|
|
try:
|
|
mentions = [
|
|
{"field": "material", "variants": ["میلگرد ۱۴"]},
|
|
{"field": "material", "variants": ["لوله ۶ اینچ"]},
|
|
]
|
|
res = await matching.resolve_mentions(
|
|
ctx, {"items": []}, mentions, trace_id="t_arb",
|
|
raw_input="استعلام میلگرد ۱۴ و لوله ۶ اینچ",
|
|
)
|
|
|
|
check("Two-stage arbitration called when Jev succeeds", len(arbiter_called_with) == 1)
|
|
passed_items = arbiter_called_with[0][1].get("items") or arbiter_called_with[0][0][2]
|
|
m0_probs = passed_items[0]["jev_probabilities"]
|
|
check("Zero probability items filtered out", "تسمه فلزی" not in m0_probs)
|
|
check("Non-zero probability preserved exactly", m0_probs.get("میلگرد ۱۴ آجدار") == 0.95)
|
|
check("_none with non-zero probability preserved", m0_probs.get("_none") == 0.05)
|
|
check("Catalog candidates list complete and untouched", len(passed_items[0]["candidates"]) > 0)
|
|
check("Arbiter endorsed valid Jev choice as exact", res[normalize_text("میلگرد ۱۴")]["status"] == "exact")
|
|
check("Arbiter vetoed wrong Jev choice to new", res[normalize_text("لوله ۶ اینچ")]["status"] == "new" and res[normalize_text("لوله ۶ اینچ")]["id"] is None)
|
|
finally:
|
|
settings.TYPESAFE_API_KEY = saved_key
|
|
matching.jev_ask = orig_jev_ask
|
|
matching.adjudicate_with_deepseek = orig_arbiter
|
|
|
|
|
|
async def test_defensive_failure_guard() -> None:
|
|
"""3. Universal Fail-Stop Guard: If DeepSeek (fallback or arbiter or NLU) fails, MatchingFailureError is raised."""
|
|
ctx = SimpleNamespace(resolver=FakeResolver())
|
|
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result=None)
|
|
|
|
async def boom_fallback(*args, **kwargs):
|
|
raise MatchingFailureError("DeepSeek API 500 error")
|
|
|
|
orig_deepseek = matching.match_with_deepseek
|
|
matching.match_with_deepseek = boom_fallback
|
|
|
|
try:
|
|
mentions = [{"field": "material", "variants": ["سیمان"]}]
|
|
failed = False
|
|
try:
|
|
await matching.resolve_mentions(ctx, {"items": []}, mentions, trace_id="t2")
|
|
except MatchingFailureError:
|
|
failed = True
|
|
check("Crisis fallback failure raises MatchingFailureError (stops cleanly)", failed)
|
|
finally:
|
|
matching.match_with_deepseek = orig_deepseek
|
|
|
|
# Also test Arbiter failure raises MatchingFailureError
|
|
matching.jev_ask = lambda state, qs: asyncio.sleep(0.001, result={"answers": {"mention_0": {"choice": "m1", "confidence": 0.9}}})
|
|
async def boom_arbiter(*args, **kwargs):
|
|
raise MatchingFailureError("Arbiter DeepSeek timeout")
|
|
|
|
orig_arbiter = matching.adjudicate_with_deepseek
|
|
matching.adjudicate_with_deepseek = boom_arbiter
|
|
try:
|
|
failed_arb = False
|
|
try:
|
|
await matching.resolve_mentions(ctx, {"items": []}, mentions, trace_id="t2_arb")
|
|
except MatchingFailureError:
|
|
failed_arb = True
|
|
check("Arbiter failure raises MatchingFailureError (stops cleanly)", failed_arb)
|
|
finally:
|
|
matching.adjudicate_with_deepseek = orig_arbiter
|
|
|
|
# Test NLUFailureError is subclass of Exception
|
|
from app.nlu import NLUFailureError
|
|
check("NLUFailureError is subclass of Exception", issubclass(NLUFailureError, Exception))
|
|
|
|
|
|
async def test_quote_resolve_payload_zero_difflib() -> None:
|
|
"""3. resolve_payload in quote.py respects Jev/fallback map and never runs difflib."""
|
|
ctx = SimpleNamespace(resolver=FakeResolver())
|
|
payload = {
|
|
"supplier_name": "تأمین‌کننده البرز",
|
|
"items": [
|
|
{"material_name": "لوله ۶ اینچ", "unit_name": "شاخه"},
|
|
{"material_name": "میلگرد ۱۴", "unit_name": "شاخه"},
|
|
],
|
|
}
|
|
jev_map = {
|
|
normalize_text("تأمین‌کننده البرز"): {
|
|
"field": "supplier", "status": "new", "variants": ["تأمین‌کننده البرز"],
|
|
"inferred_category_slug": "metals-steel-sections",
|
|
},
|
|
normalize_text("لوله ۶ اینچ"): {
|
|
"field": "material", "status": "new", "variants": ["لوله ۶ اینچ"],
|
|
},
|
|
normalize_text("میلگرد ۱۴"): {
|
|
"field": "material", "status": "exact", "id": "m1", "name": "میلگرد ۱۴ آجدار",
|
|
"variants": ["میلگرد ۱۴"],
|
|
},
|
|
normalize_text("شاخه"): {
|
|
"field": "unit", "status": "exact", "id": "u1", "name": "شاخه", "variants": ["شاخه"],
|
|
},
|
|
}
|
|
|
|
await quote.resolve_payload(ctx, payload, trace_id="t3", jev_map=jev_map)
|
|
|
|
check("New supplier kept id=None (no difflib forced match)", payload.get("supplier_id") is None)
|
|
check("New supplier inferred category saved", payload.get("supplier_inferred_category_slug") == "metals-steel-sections")
|
|
check("New material kept id=None (no difflib forced match to 4-inch pipe)", payload["items"][0].get("material_id") is None)
|
|
check("Exact material bound to catalog ID", payload["items"][1].get("material_id") == "m1")
|
|
check("Exact unit bound to catalog ID", payload["items"][0].get("unit_id") == "u1")
|
|
|
|
|
|
async def test_quote_commit_defensive_get_or_create() -> None:
|
|
"""4. commit in quote.py uses get_or_create to prevent 409 conflict errors."""
|
|
api = FakeApi()
|
|
resolver = FakeResolver()
|
|
ctx = SimpleNamespace(api=api, resolver=resolver, pending={},
|
|
send=lambda *a, **k: asyncio.sleep(0.0001))
|
|
user = SimpleNamespace(actor_id="usr_1")
|
|
|
|
# Mock obs calls so offline test doesn't require live DB connection
|
|
orig_latest = quote.obs.latest_trace_for_draft
|
|
orig_link = quote.obs.link_draft
|
|
orig_event = quote.obs.event
|
|
quote.obs.latest_trace_for_draft = lambda did: asyncio.sleep(0.0001, result="mock_trace")
|
|
quote.obs.link_draft = lambda *a, **k: asyncio.sleep(0.0001)
|
|
quote.obs.event = lambda *a, **k: asyncio.sleep(0.0001)
|
|
|
|
# Item with name "میلگرد ۱۴ آجدار" that already exists in DB
|
|
payload = {
|
|
"supplier_name": "فولاد خوزستان", # Already in DB
|
|
"supplier_inferred_category_slug": "metals-steel-sections",
|
|
"items": [
|
|
{"material_name": "میلگرد ۱۴ آجدار", "unit_id": 1, "quantity": 10, "unit_price": 25000},
|
|
{"material_name": "تیرآهن ۱۸", "unit_id": 1, "quantity": 5, "unit_price": 50000}, # Brand new
|
|
],
|
|
"project_name": "پروژه پردیس", # Already in DB
|
|
}
|
|
draft = SimpleNamespace(id="d1", payload=payload, review_message_id=None)
|
|
|
|
# Mock create_quote
|
|
api.create_quote = lambda body, actor_id: asyncio.sleep(0.001, result={"id": "q1"})
|
|
# Mock draft_store.set_status
|
|
quote.draft_store.set_status = lambda did, st: asyncio.sleep(0.001)
|
|
|
|
try:
|
|
await quote.commit(ctx, chat_id=123, user=user, draft=draft, only_resolved=False)
|
|
|
|
check("Existing supplier reused ID from DB (not recreated)", payload["supplier_id"] == "s1")
|
|
check("Existing material reused ID from DB on conflict (not crashing)", payload["items"][0]["material_id"] == "m1")
|
|
check("New material created cleanly", payload["items"][1]["material_id"] == "m_new_1")
|
|
check("Existing project reused ID from DB", payload["project_id"] == "p1")
|
|
finally:
|
|
quote.obs.latest_trace_for_draft = orig_latest
|
|
quote.obs.link_draft = orig_link
|
|
quote.obs.event = orig_event
|
|
|
|
|
|
async def main() -> None:
|
|
print("Running DeepSeek Crisis Fallback & Zero-Difflib Tests...\n")
|
|
orig_fb_event = fallback_matcher.obs.event
|
|
fallback_matcher.obs.event = lambda *a, **k: asyncio.sleep(0.0001)
|
|
try:
|
|
await test_jev_failure_triggers_fallback()
|
|
await test_two_stage_adjudication_flow()
|
|
await test_defensive_failure_guard()
|
|
await test_quote_resolve_payload_zero_difflib()
|
|
await test_quote_commit_defensive_get_or_create()
|
|
finally:
|
|
fallback_matcher.obs.event = orig_fb_event
|
|
|
|
total = len(PASSED) + len(FAILED)
|
|
print(f"\n{len(PASSED)}/{total} passed")
|
|
if FAILED:
|
|
print("FAILED:", FAILED)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|