feat(matching): Jev entity matching phases 2-5 (per-field fan-out, variants, candidate ambiguity buttons)
Phase 2 — jev.py client: POST /v1/systemone, PINNED model jev-1.13.0 (drift warning when the response model differs), one retry on 429/5xx, fail-soft (None on any error), usage/latency returned for observability. Config adds TYPESAFE_API_KEY / JEV_* (thresholds 0.85/0.60, 15 questions per field); no daily cap by owner decision. bot/.env is now mounted read-only into the container so the key never enters compose/git.
Phase 3 — matching.py: برونگروهی (one request per field, parallel), درونگروهی (one choice question per mention sharing the same full candidate list + «_none»), درونواریانسی (all STT spellings in one question). Routing: conf>=exact → bind; floor..exact → ambiguity; else/_none → new; missing/failed field → difflib fallback (bug fixed during testing: a failed request used to record 'new' instead of falling back). Resolver gains full cached lists for suppliers/materials (paged) plus brands (API+DB) and specs (EAV values + draft specs). NLU prompt now emits entity_mentions (6 fields) with the voice-variant grouping rule; sanitize_mentions guards the shape. handle_message runs matching only for register_quote/update_draft/add_material behind JEV_ENABLED; jev_map threads into quote/material resolve_payload (Jev hit skips difflib; exact writes ids; ambiguity skips difflib). Spec effect per owner: spelling normalization only (normalize_brand_spec).
Phase 4 — entity-match ambiguity sub-messages: create_ambiguities() builds deduped entries {kind: entity_match, field, variants, candidates, details with Persian percentages}; new keyboards.ambiguity_candidates_keyboard renders one button per candidate + «هیچکدام — ایجاد جدید»; callbacks handle amb⛏️{prefix}:{amb_id}:{idx} (applies the candidate via matching.apply_choice then re-resolves/renders) and amb:new (rejected → auto-create path at commit). Every user choice lands in observability (ambiguity.user_choice with arg+candidates).
Phase 5 — voice variants: NLU groups different STT spellings of one entity into one variants[] (verified live: مصطفوی/مستفدی → one supplier mention); tools/test_matching.py live section (RUN_LIVE=1) exercises real DeepSeek + real Jev without touching Bale.
Tests: test_jev.py 5/5 (mock HTTP: no-key, happy path, 429-retry, 400, exception), test_matching.py 11/11 mock + 3/3 live (real NLU produced 7 mentions with correct fields; real Jev matched «50 کیلویی»→«۵۰ کیلویی» 0.98 and «کیسه» 1.0 and correctly returned new against the emptied catalog); observability regression 8/8.
This commit is contained in:
@@ -153,6 +153,43 @@ async def _db_materials(q: str | None = None) -> list[Candidate]:
|
||||
return []
|
||||
|
||||
|
||||
async def _db_brands() -> list[Candidate]:
|
||||
try:
|
||||
async with SessionLocal() as s:
|
||||
rows = (await s.execute(text("SELECT id, name FROM brands"))).mappings().all()
|
||||
return [Candidate(id=str(b["id"]), name=b.get("name") or str(b["id"])) for b in rows]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
async def _db_specs() -> list[Candidate]:
|
||||
"""Distinct spec values: EAV strings from the catalog + specs used in recent drafts."""
|
||||
values: dict[str, None] = {}
|
||||
try:
|
||||
async with SessionLocal() as s:
|
||||
rows = (await s.execute(text(
|
||||
"SELECT DISTINCT value_string FROM material_attribute_values "
|
||||
"WHERE value_string IS NOT NULL AND btrim(value_string) <> '' LIMIT 500"
|
||||
))).scalars().all()
|
||||
for v in rows:
|
||||
values.setdefault(str(v), None)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
async with SessionLocal() as s:
|
||||
rows = (await s.execute(text(
|
||||
"SELECT DISTINCT spec FROM bot_drafts d, "
|
||||
"jsonb_array_elements(COALESCE(d.payload->'items', '[]'::jsonb)) it, "
|
||||
"jsonb_array_elements_text(COALESCE(it->'specs', '[]'::jsonb)) spec "
|
||||
"WHERE btrim(spec) <> '' LIMIT 300"
|
||||
))).scalars().all()
|
||||
for v in rows:
|
||||
values.setdefault(str(v), None)
|
||||
except Exception:
|
||||
pass
|
||||
return [Candidate(id=v, name=v) for v in values]
|
||||
|
||||
|
||||
class Resolver:
|
||||
"""Caches catalog lists (units/categories/projects are stable; suppliers/materials queried on demand)."""
|
||||
|
||||
@@ -236,6 +273,74 @@ class Resolver:
|
||||
items = await _db_materials(q=q)
|
||||
return _rank(q, items) if q else items
|
||||
|
||||
# ------------------------------------------------- full lists (Jev, v5)
|
||||
|
||||
async def _fetch_all(self, fetch_page, max_pages: int = 5) -> list[dict]:
|
||||
"""Page through a paginated ERP list endpoint (page_size=100)."""
|
||||
out: list[dict] = []
|
||||
for page in range(1, max_pages + 1):
|
||||
rows = await fetch_page(page)
|
||||
out += rows
|
||||
if len(rows) < 100:
|
||||
break
|
||||
return out
|
||||
|
||||
async def all_suppliers(self) -> list[Candidate]:
|
||||
async def fetch():
|
||||
try:
|
||||
rows = await self._fetch_all(
|
||||
lambda page: self._api.list_suppliers(page=page, page_size=100)
|
||||
)
|
||||
return [
|
||||
Candidate(
|
||||
id=str(s["id"]),
|
||||
name=s.get("name") or str(s["id"]),
|
||||
extra={
|
||||
"location": s.get("location"),
|
||||
"phone": s.get("phone"),
|
||||
"contact_person": s.get("contact_person"),
|
||||
"category_name": (s.get("category") or {}).get("name")
|
||||
if isinstance(s.get("category"), dict) else s.get("category_name"),
|
||||
},
|
||||
)
|
||||
for s in rows
|
||||
]
|
||||
except Exception:
|
||||
return await _db_suppliers()
|
||||
return await self._cached("all_suppliers", fetch)
|
||||
|
||||
async def all_materials(self) -> list[Candidate]:
|
||||
async def fetch():
|
||||
try:
|
||||
rows = await self._fetch_all(
|
||||
lambda page: self._api.list_materials(page=page, page_size=100)
|
||||
)
|
||||
return [
|
||||
Candidate(
|
||||
id=str(m["id"]),
|
||||
name=m.get("name") or str(m["id"]),
|
||||
extra={"canonical_unit_id": m.get("canonical_unit_id")},
|
||||
)
|
||||
for m in rows
|
||||
]
|
||||
except Exception:
|
||||
return await _db_materials()
|
||||
return await self._cached("all_materials", fetch)
|
||||
|
||||
async def brands(self) -> list[Candidate]:
|
||||
async def fetch():
|
||||
try:
|
||||
rows = await self._fetch_all(
|
||||
lambda page: self._api.list_brands(page=page, page_size=100)
|
||||
)
|
||||
return [Candidate(id=str(b["id"]), name=b.get("name") or str(b["id"])) for b in rows]
|
||||
except Exception:
|
||||
return await _db_brands()
|
||||
return await self._cached("brands", fetch)
|
||||
|
||||
async def specs(self) -> list[Candidate]:
|
||||
return await self._cached("specs", _db_specs)
|
||||
|
||||
# ------------------------------------------------------------ resolutions
|
||||
|
||||
async def resolve(self, kind: str, name: str | None) -> Resolution:
|
||||
|
||||
Reference in New Issue
Block a user