83 lines
4.1 KiB
Python
83 lines
4.1 KiB
Python
"""تست‌های خالص (بدون مدل/دیتابیس) برای ماژول‌های چت‌بات — اجرا: python test_pure.py"""
|
|
import sys
|
|
import os
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
from chatbot.content import chunk_documents, tiptap_blocks, body_to_text
|
|
from chatbot.vectorstore import chunk_content_hash, chatbot_dsn
|
|
from chatbot.persian import normalize, tokenize
|
|
|
|
docs = [
|
|
{"text": "پاراگراف کوتاه خدمت الف", "type": "service", "title": "الف", "url": "/services"},
|
|
{"text": "سؤال اول؟", "type": "faq", "title": "س۱", "url": "/faq"},
|
|
{"text": "سؤال دوم؟", "type": "faq", "title": "س۲", "url": "/faq"},
|
|
{"text": "آ" * 2000, "type": "article", "title": "مقاله بلند", "url": "/blog/long"},
|
|
]
|
|
chunks = chunk_documents(docs)
|
|
ids = [c["doc_id"] for c in chunks]
|
|
assert len(ids) == len(set(ids)), f"duplicate ids: {ids}"
|
|
|
|
faq_ids = sorted(c["doc_id"] for c in chunks if c["type"] == "faq")
|
|
assert faq_ids == ["faq:faq:0:1", "faq:faq:1:1"], faq_ids
|
|
|
|
# سندهایی که url یکتا دارند (پروژه/مقاله) نباید روی هم بیفتند — url بخشی از doc_id است
|
|
two_projects = [
|
|
{"text": "پروژه الف " + "ت" * 60, "type": "project", "title": "الف", "url": "/projects/alpha"},
|
|
{"text": "پروژه ب " + "ت" * 60, "type": "project", "title": "ب", "url": "/projects/beta"},
|
|
]
|
|
pids = sorted(c["doc_id"] for c in chunk_documents(two_projects))
|
|
assert pids == ["project:projects-alpha:0:1", "project:projects-beta:0:1"], pids
|
|
|
|
# تیتر سندهای تک‌قطعه‌ای باید در متن امبدشده باشد تا جست‌وجو پیدایش کند
|
|
faq_chunks = [c for c in chunks if c["type"] == "faq"]
|
|
assert all(c["text"].startswith(c["title"]) for c in faq_chunks), [c["text"] for c in faq_chunks]
|
|
|
|
art = [c for c in chunks if c["type"] == "article"]
|
|
assert len(art) >= 2 and art[1]["doc_id"] == "article:blog-long:0:2", [x["doc_id"] for x in art]
|
|
assert "بخش 2 از" in art[1]["text"] and "بخش 1 از" in art[0]["text"]
|
|
|
|
c = chunks[0]
|
|
assert c["content_hash"] == chunk_content_hash(c)
|
|
c2 = dict(c, title="عنوان عوض شد")
|
|
# hash فیلد کپی‌شده است — باید با محاسبه دوباره مقایسه شود، نه با فیلد قدیمی
|
|
assert chunk_content_hash(c2) != c["content_hash"]
|
|
|
|
chunks2 = chunk_documents(docs)
|
|
assert [x["doc_id"] for x in chunks2] == ids
|
|
assert [x["content_hash"] for x in chunks2] == [x["content_hash"] for x in chunks]
|
|
|
|
settings_dsn = chatbot_dsn()
|
|
assert settings_dsn.endswith("/aramesh_chatbot"), settings_dsn
|
|
assert "+psycopg" not in settings_dsn
|
|
|
|
# نرمال‌سازی و توکن‌سازی فارسی
|
|
assert normalize("كامل يك ۱۲۳") == "کامل یک 123"
|
|
t = tokenize("کدام شرکت ساختمانی خوب در تهران؟")
|
|
assert "شرکت" in t and "تهران" in t and "کدام" not in t, t
|
|
assert "می\u200cسازیم" in tokenize("می\u200cسازیم") and "سازیم" in tokenize("می\u200cسازیم")
|
|
|
|
# استخراج متن TipTap و فرمت قدیمی
|
|
tiptap_doc = {
|
|
"type": "doc",
|
|
"content": [
|
|
{"type": "heading", "attrs": {"level": 2}, "content": [{"type": "text", "text": "معرفی پروژه"}]},
|
|
{"type": "paragraph", "content": [
|
|
{"type": "text", "text": "این برج در تهران "},
|
|
{"type": "text", "marks": [{"type": "bold"}], "text": "ساخته شد"},
|
|
{"type": "hardBreak"},
|
|
{"type": "image", "attrs": {"alt": "نمای بیرونی"}},
|
|
]},
|
|
{"type": "bulletList", "content": [
|
|
{"type": "listItem", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "۲۴ طبقه"}]}]},
|
|
]},
|
|
{"type": "paragraph", "content": [{"type": "image", "attrs": {"alt": ""}}]},
|
|
],
|
|
}
|
|
blocks = tiptap_blocks(tiptap_doc)
|
|
assert blocks == ["معرفی پروژه", "این برج در تهران ساخته شد [تصویر: نمای بیرونی]", "۲۴ طبقه"], blocks
|
|
assert body_to_text(["الف", "ب"]) == "الف\nب"
|
|
assert body_to_text(None) == ""
|
|
|
|
print(f"ALL PURE TESTS OK — {len(chunks)} chunks from {len(docs)} docs")
|