feat(bot): phase-6 calibration — mock seed, A/B harness, low-trust fallback
- tools/seed_mock.py: idempotent additive mock data (uuid5 ids) covering all tables (units, brands, specs, suppliers, materials + EAV/brand/unit links, projects, warehouses, inventory, quotes with termsheets/line items/decisions, receipts) so phases 6-7 can be exercised end-to-end - tools/calibrate_matching.py: A/B difflib vs Jev on 35 scenarios (31 positive + 4 negative); collects RAW picks with widened bands, classifies against the configured thresholds, sweeps EXACT 0.55-0.95 (negatives count as binds), recommends FLOOR strictly above the highest wrong pick with margin for ±0.05 run-to-run confidence jitter; markdown report + real traces - matching.py: below-FLOOR picks no longer suppress difflib — they log a routing.decision event (source=jev-lowtrust) and leave no result so the existing difflib path keeps its chance (0.54 «پردیس» was lost to «new») - config.py: JEV_CONF_FLOOR 0.60 -> 0.65 (removes the جعبه→بسته @0.61 wrong candidate offer; correct sub-floor picks باق اسپارت/کیلو are difflib-covered) - test_matching.py: +2 low-trust cases (13/13 mock, 16/16 live) - docker-compose.yml: mount bot/tools into the bot container
This commit is contained in:
+3
-1
@@ -147,7 +147,9 @@ class Settings(BaseSettings):
|
||||
JEV_ENABLED: bool = True
|
||||
JEV_TIMEOUT: float = 8.0
|
||||
JEV_CONF_EXACT: float = 0.85
|
||||
JEV_CONF_FLOOR: float = 0.60
|
||||
JEV_CONF_FLOOR: float = 0.65 # phase-6 calibration (2026-10-12): 0.60 let one
|
||||
# wrong candidate (جعبه→بسته @0.61) through; the
|
||||
# correct picks it drops are covered by difflib.
|
||||
JEV_MAX_QUESTIONS_PER_FIELD: int = 15
|
||||
|
||||
LOG_LEVEL: str = "INFO"
|
||||
|
||||
+20
-8
@@ -168,18 +168,30 @@ async def resolve_mentions(ctx, patch: dict, mentions: list[dict], trace_id) ->
|
||||
key=lambda x: -x["prob"],
|
||||
)[:6]
|
||||
|
||||
base = {"field": field, "variants": m["variants"], "confidence": conf,
|
||||
"choice": (cand.name if cand is not None else None),
|
||||
"candidates": ranked}
|
||||
|
||||
if cand is not None and conf >= settings.JEV_CONF_EXACT:
|
||||
res = {"field": field, "status": "exact", "variants": m["variants"],
|
||||
"id": cand.id, "name": cand.name, "extra": cand.extra or {},
|
||||
"confidence": conf}
|
||||
res = {**base, "status": "exact", "id": cand.id, "name": cand.name,
|
||||
"extra": cand.extra or {}}
|
||||
elif cand is not None and conf >= settings.JEV_CONF_FLOOR:
|
||||
res = {"field": field, "status": "ambiguity", "variants": m["variants"],
|
||||
"id": cand.id, "name": cand.name, "extra": cand.extra or {},
|
||||
"confidence": conf,
|
||||
res = {**base, "status": "ambiguity", "id": cand.id, "name": cand.name,
|
||||
"extra": cand.extra or {},
|
||||
"candidates": ranked or [{"id": cand.id, "name": cand.name, "prob": conf}]}
|
||||
elif cand is None:
|
||||
# Explicit «_none» (or an unusable choice): the model judged the whole
|
||||
# list → treat as new (the auto-create path handles it).
|
||||
res = {**base, "status": "new"}
|
||||
else:
|
||||
res = {"field": field, "status": "new", "variants": m["variants"],
|
||||
"confidence": conf, "candidates": ranked}
|
||||
# Picked something but below the floor = low trust: leave NO result so
|
||||
# the difflib fallback still gets its chance (suppressing it here would
|
||||
# let a correct-but-under-confident pick create duplicates instead).
|
||||
await obs.event(trace_id, "routing.decision", field=field, payload={
|
||||
"source": "jev-lowtrust", "variants": m["variants"],
|
||||
"choice": cand.name, "confidence": conf,
|
||||
})
|
||||
continue
|
||||
for v in m["variants"]:
|
||||
results[normalize_text(v)] = res
|
||||
decisions.append({"variants": m["variants"], "status": res["status"],
|
||||
|
||||
Reference in New Issue
Block a user