feat(stt): support multi-model STT with parallel execution and consensus NLU extraction
This commit is contained in:
@@ -14,8 +14,8 @@ from app import drafts as draft_store
|
||||
from app.config import settings
|
||||
from app.nlu import understand
|
||||
from app.runtime import BotContext, LinkedUser
|
||||
from app.stt import transcribe
|
||||
from app.texts import CANCELLED, LINK_START, NOTHING_TO_COMMIT, STT_FAILED, UNKNOWN_HINT, VOICE_EMPTY
|
||||
from app.stt import transcribe, transcribe_multi
|
||||
from app.texts import CANCELLED, LINK_START, NOTHING_TO_COMMIT, STT_FAILED, UNKNOWN_HINT, VOICE_EMPTY, esc
|
||||
|
||||
from . import commands, callbacks, quote, supplier, material, queries, decisions
|
||||
|
||||
@@ -40,6 +40,7 @@ async def handle_update(ctx: BotContext, update: dict) -> None:
|
||||
async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
|
||||
text = (message.get("text") or "").strip()
|
||||
source = "text"
|
||||
voice_transcripts: list[dict] | None = None
|
||||
|
||||
if text.startswith("/"):
|
||||
await commands.handle_command(ctx, chat_id, text)
|
||||
@@ -51,17 +52,20 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
|
||||
file_id = (message.get("voice") or message.get("audio") or message.get("document"))["file_id"]
|
||||
try:
|
||||
audio = await ctx.bale.download_file(file_id)
|
||||
text = await transcribe(settings, audio,
|
||||
mime_type=(message.get("voice") or {}).get("mime_type"),
|
||||
filename=(message.get("voice") or {}).get("file_name"))
|
||||
voice_transcripts = await transcribe_multi(
|
||||
settings,
|
||||
audio,
|
||||
mime_type=(message.get("voice") or {}).get("mime_type"),
|
||||
filename=(message.get("voice") or {}).get("file_name"),
|
||||
)
|
||||
except Exception as exc:
|
||||
detail = "هوش مصنوعی تنظیم نشده" if "not configured" in str(exc) else str(exc)[:200]
|
||||
await ctx.send(chat_id, STT_FAILED.format(detail=esc(detail)))
|
||||
return
|
||||
if not text:
|
||||
if not voice_transcripts:
|
||||
await ctx.send(chat_id, VOICE_EMPTY)
|
||||
return
|
||||
await ctx._log(chat_id, "in", "voice", text)
|
||||
text = voice_transcripts[0]["text"]
|
||||
|
||||
# Pending multi-step interactions take the message first.
|
||||
flow = ctx.pending.get(chat_id)
|
||||
@@ -91,9 +95,16 @@ async def handle_message(ctx: BotContext, chat_id: int, message: dict) -> None:
|
||||
return
|
||||
|
||||
draft_summary = _draft_summary(target_draft) if target_draft else None
|
||||
result = await understand(settings, text, draft_summary)
|
||||
input_payload = voice_transcripts if voice_transcripts else text
|
||||
result = await understand(settings, input_payload, draft_summary)
|
||||
intent, patch = result.get("intent"), result.get("patch") or {}
|
||||
logger.info("chat %s intent=%s conf=%s (is_edit=%s)", chat_id, intent, result.get("confidence"), target_draft is not None)
|
||||
consensus_text = result.get("consensus_text") or text
|
||||
|
||||
if source == "voice":
|
||||
await ctx._log(chat_id, "in", "voice", consensus_text)
|
||||
|
||||
logger.info("chat %s intent=%s conf=%s consensus=%r (is_edit=%s)",
|
||||
chat_id, intent, result.get("confidence"), consensus_text[:40], target_draft is not None)
|
||||
|
||||
if intent in ("register_quote", "update_draft"):
|
||||
await quote.handle_entities(
|
||||
|
||||
Reference in New Issue
Block a user