feat(bot): phase-6 calibration — mock seed, A/B harness, low-trust fallback
- tools/seed_mock.py: idempotent additive mock data (uuid5 ids) covering all tables (units, brands, specs, suppliers, materials + EAV/brand/unit links, projects, warehouses, inventory, quotes with termsheets/line items/decisions, receipts) so phases 6-7 can be exercised end-to-end - tools/calibrate_matching.py: A/B difflib vs Jev on 35 scenarios (31 positive + 4 negative); collects RAW picks with widened bands, classifies against the configured thresholds, sweeps EXACT 0.55-0.95 (negatives count as binds), recommends FLOOR strictly above the highest wrong pick with margin for ±0.05 run-to-run confidence jitter; markdown report + real traces - matching.py: below-FLOOR picks no longer suppress difflib — they log a routing.decision event (source=jev-lowtrust) and leave no result so the existing difflib path keeps its chance (0.54 «پردیس» was lost to «new») - config.py: JEV_CONF_FLOOR 0.60 -> 0.65 (removes the جعبه→بسته @0.61 wrong candidate offer; correct sub-floor picks باق اسپارت/کیلو are difflib-covered) - test_matching.py: +2 low-trust cases (13/13 mock, 16/16 live) - docker-compose.yml: mount bot/tools into the bot container
This commit is contained in:
@@ -63,7 +63,7 @@ async def mock_tests() -> None:
|
||||
ctx = SimpleNamespace(resolver=FakeResolver())
|
||||
orig_ask = matching.jev_ask
|
||||
|
||||
# 1) routing bands: exact / ambiguity / new + variant grouping
|
||||
# 1) routing bands: exact / ambiguity / low-trust / new + variant grouping
|
||||
matching.jev_ask = make_fake_ask({
|
||||
"mention_0": {"choice": "منشانه", "confidence": 0.9,
|
||||
"probabilities": {"منشانه": 0.95, "_none": 0.05}},
|
||||
@@ -71,11 +71,14 @@ async def mock_tests() -> None:
|
||||
"probabilities": {"دفتر": 0.73, "منشانه": 0.15, "_none": 0.12}},
|
||||
"mention_2": {"choice": "_none", "confidence": 0.9,
|
||||
"probabilities": {"_none": 0.94}},
|
||||
"mention_3": {"choice": "دفتر", "confidence": 0.5,
|
||||
"probabilities": {"دفتر": 0.52, "_none": 0.48}},
|
||||
})
|
||||
mentions = [
|
||||
{"field": "project", "variants": ["منشیانه"]},
|
||||
{"field": "project", "variants": ["دفتر", "دفر"]},
|
||||
{"field": "project", "variants": ["چیز عجیب"]},
|
||||
{"field": "project", "variants": ["دفر دو"]},
|
||||
]
|
||||
res = await matching.resolve_mentions(ctx, {"items": []}, mentions, None)
|
||||
r0 = matching.lookup(res, "منشیانه")
|
||||
@@ -87,6 +90,9 @@ async def mock_tests() -> None:
|
||||
check("variant of same mention shares one result", r1 is r0 if False else
|
||||
(r1 and matching.lookup(res, "دفتر") is r1))
|
||||
check("_none / low conf -> new", r2 and r2["status"] == "new")
|
||||
check("below-floor pick -> NO result (difflib fallback keeps its chance)",
|
||||
matching.lookup(res, "دفر دو") is None)
|
||||
check("below-floor pick did not pollute other mentions", r1 and r1["status"] == "ambiguity")
|
||||
|
||||
# 2) Jev failure -> empty map (callers fall back to difflib)
|
||||
matching.jev_ask = make_fake_ask({})
|
||||
|
||||
Reference in New Issue
Block a user