用测试和评测检查知识库
分别验证接口行为和检索结果,保留失败样本,并用固定问题比较修改。
一个问题回答正确,还不足以说明其他问题也正确。我们需要把问题保存下来,每次修改代码或资料后重新运行。
测试检查约定的程序行为,例如没有身份时返回401、删除后查不到文档。评测关注任务效果,例如问题是否找到了正确依据。两者放在一起,既能发现代码故障,也能观察检索效果。
先检查可重复的行为
在项目中创建 tests 文件夹。Windows与macOS终端都可以执行:
mkdir tests保存下面文件为 tests/test_api.py:
"""使用临时磁盘库验证真实 SQL 和完整 HTTP 路由。"""
import pytest
import json
import httpx
from fastapi.testclient import TestClient
from main import create_app
from provider import Generator
ALPHA = {"Authorization": "Bearer demo-alpha-token"}
BETA = {"Authorization": "Bearer demo-beta-token"}
def test_trace_links_response_without_copying_credentials(tmp_path, monkeypatch):
# 根据同一个请求编号关联响应和记录,检查日志字段的范围。
trace = tmp_path / "trace.jsonl"
monkeypatch.setenv("KB_TRACE_PATH", str(trace))
with TestClient(create_app(tmp_path / "trace.sqlite3", enable_generation=False)) as client:
response = client.post("/ask", headers=ALPHA, json={"query": "私人问题"})
row = json.loads(trace.read_text(encoding="utf-8").strip())
assert row["request_id"] == response.headers["X-Request-ID"]
assert row["status"] == 200 and row["duration_ms"] >= 0
assert set(row) == {"request_id", "method", "status", "duration_ms"}
assert response.headers["Server-Timing"].startswith("app;dur=")
def test_evaluation_forces_generation_off(tmp_path, monkeypatch):
# 即使上一节开启了模型,离线检查也不会读取密钥或请求网络。
monkeypatch.setenv("KB_GENERATION_ENABLED", "1")
monkeypatch.delenv("KB_MODEL_API_KEY", raising=False)
with TestClient(create_app(tmp_path / "offline.sqlite3", enable_generation=False)) as client:
put(client)
assert ask(client).json()["mode"] == "extractive"
@pytest.fixture
def client(tmp_path, monkeypatch):
monkeypatch.delenv("KB_GENERATION_ENABLED", raising=False)
with TestClient(create_app(tmp_path / "kb.sqlite3")) as active:
yield active
def put(client, text="差旅报销须30天内提交。", headers=ALPHA):
return client.put("/documents/rule", headers=headers, json={"title": "制度", "text": text})
def ask(client, query="报销", headers=ALPHA):
return client.post("/ask", headers=headers, json={"query": query})
def test_health_auth(client):
assert client.get("/health").json()["status"] == "ok"
assert client.post("/search", json={"query": "报销"}).status_code == 401
assert put(client, headers={"Authorization": "Bearer fake"}).status_code == 401
def test_crud_index_and_citations(client):
assert put(client).status_code == 200
result = ask(client).json()
assert result["mode"] == "extractive"
assert result["citations"][0]["chunk_id"] == "rule:0"
assert "30天" in result["answer"]
assert put(client, "休假需要提前申请。").status_code == 200
assert ask(client).json()["mode"] == "abstain"
assert ask(client, "休假").json()["mode"] == "extractive"
assert client.delete("/documents/rule", headers=ALPHA).status_code == 200
assert ask(client, "休假").json()["mode"] == "abstain"
assert client.delete("/documents/rule", headers=ALPHA).status_code == 404
def test_tenant_isolation(client):
put(client)
assert ask(client, headers=BETA).json()["mode"] == "abstain"
assert client.delete("/documents/rule", headers=BETA).status_code == 404
put(client, "报销期限为7天。", BETA)
assert "30天" in ask(client).json()["answer"]
assert "7天" in ask(client, headers=BETA).json()["answer"]
assert client.put("/documents/x", headers=ALPHA,
json={"title": "制度", "text": "正文", "tenant": "beta"}).status_code == 422
@pytest.mark.parametrize("query", ["", " "])
def test_empty_query(client, query):
assert ask(client, query).status_code == 422
def test_unknown_and_punctuation(client):
put(client)
for query in ["火星移民", "! * OR ( )"]:
assert ask(client, query).json()["mode"] == "abstain"
def test_persistence(tmp_path, monkeypatch):
monkeypatch.delenv("KB_GENERATION_ENABLED", raising=False)
path = tmp_path / "persistent.sqlite3"
with TestClient(create_app(path)) as first:
put(first)
with TestClient(create_app(path)) as second:
assert "30天" in ask(second).json()["answer"]
@pytest.mark.parametrize("result", [
{"answer": "泄露内容", "citation_ids": ["other:0"]},
{"answer": "没证据", "citation_ids": []},
{"answer": "格式错误", "citation_ids": [{}]},
[], {"answer": "", "citation_ids": ["rule:0"]},
])
def test_reject_generated_citations(tmp_path, result):
with TestClient(create_app(tmp_path / "mock.sqlite3", lambda q, h: result)) as client:
put(client)
assert ask(client).json()["reason"] == "invalid_generation"
def test_accept_generated_and_provider_failure(tmp_path):
def good(query, hits):
return {"answer": "应在30天内提交。", "citation_ids": [hits[0]["chunk_id"]]}
with TestClient(create_app(tmp_path / "good.sqlite3", good)) as client:
put(client)
assert ask(client).json()["mode"] == "generated"
def failed(query, hits):
raise httpx.ConnectError("private upstream detail")
with TestClient(create_app(tmp_path / "bad.sqlite3", failed)) as client:
put(client)
response = ask(client)
assert response.status_code == 502
assert "private" not in response.text
def test_real_http_adapter_contract(monkeypatch):
monkeypatch.setenv("KB_MODEL_BASE_URL", "https://example.invalid/v1")
monkeypatch.setenv("KB_MODEL_API_KEY", "test-only")
monkeypatch.setenv("KB_MODEL_NAME", "test-model")
def fake_post(url, **kwargs):
assert url == "https://example.invalid/v1/chat/completions"
assert kwargs["timeout"] == 20
assert kwargs["json"]["model"] == "test-model"
return httpx.Response(200, request=httpx.Request("POST", url), json={
"choices": [{"message": {"content": '{"answer":"摘录","citation_ids":["rule:0"]}'}}]})
monkeypatch.setattr(httpx, "post", fake_post)
assert Generator()("问题", [])["citation_ids"] == ["rule:0"]TestClient把请求送进真实应用处理流程,测试期间使用临时数据库。它适合快速验证身份、输入、存储与响应;网络连接问题在上一节的真实HTTP请求中检查。
在项目根目录运行:
python -m pytest -q15 passed, 1 warning in 4.23s检查点包括:保存与查询、更新与删除、跨团队访问、无效输入、引用范围,以及模型服务失败后的响应。测试里的模型替身用来主动制造格式错误或越界引用,让这些难以稳定复现的分支每次都能检查。
再跑一组检索问题
保存为 evaluate.py:
"""隔离临时库评测固定样本;不调用模型,不改服务数据库。"""
import tempfile
from pathlib import Path
from fastapi.testclient import TestClient
from main import create_app
from seed import DOCUMENTS
def evaluate():
# 放在项目目录内,Windows 下也不会落到系统盘临时目录。
scratch = Path(__file__).parent / "data"
scratch.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=scratch) as folder:
with TestClient(create_app(Path(folder) / "eval.sqlite3", enable_generation=False)) as client:
client.headers["Authorization"] = "Bearer demo-alpha-token"
for doc_id, title, text in DOCUMENTS:
client.put(f"/documents/{doc_id}", json={"title": title, "text": text}).raise_for_status()
cases = [("差旅报销", "travel"), ("权限申请", "access"),
("生产事故", "incident"), ("火星移民", None)]
passed = 0
for query, expected in cases:
result = client.post("/ask", json={"query": query}).json()
ids = [c["document_id"] for c in result["citations"]]
ok = expected in ids if expected else result["mode"] == "abstain"
passed += ok
print(f"{query}: {'PASS' if ok else 'FAIL'}; mode={result['mode']}; ids={ids}")
print(f"固定样本通过 {passed}/{len(cases)}")
if passed != len(cases):
raise SystemExit(1)
if __name__ == "__main__":
evaluate()运行:
python evaluate.py差旅报销: PASS; mode=extractive; ids=['travel']
权限申请: PASS; mode=extractive; ids=['access']
生产事故: PASS; mode=extractive; ids=['incident']
火星移民: PASS; mode=abstain; ids=[]
固定样本通过 4/4这里真正调用了应用的入库与问答处理逻辑,再比较返回的文档编号。前三条应找到对应制度,最后一条应拒答。总数由运行结果累加得出。
从四条扩展到自己的评测集
四条问题用于确认链路跑通。要判断知识库是否适合实际使用,继续加入不同问法、缺失资料、过期资料和容易混淆的制度。
| 问题类型 | 例子 | 观察什么 |
|---|---|---|
| 原文措辞 | 差旅报销多久内提交 | 正确文档是否被找回 |
| 不同措辞 | 出差回来多久交单 | 关键词检索是否漏掉同义表达 |
| 多份资料 | 权限怎么开通,事故找谁 | 两份必要资料是否都找回 |
| 无资料 | 火星移民怎么办 | 是否说明资料不足 |
| 不同团队 | beta查询alpha制度 | 是否泄露其他团队内容 |
把一部分问题用于调整方案,另一部分保留到最终检查。记录每个失败问题的输入、找到的片段和预期依据,分别归类为“没有找回”“找错资料”“回答没有遵循依据”。
Recall@k表示需要的依据中,有多少进入前k个结果。例如需要两份制度,只找到一份,召回率为1/2。回答质量再检查每条说法与引用是否对应。接入真实模型后,还应记录模型名称、提示版本、响应、用量与耗时,便于比较。
做一次有对照的修改
保存为 compare_queries.py,运行一组明确的查询改写对照:
from storage import Store
store = Store("data/kb.sqlite3")
# 同一问题,只改变检索用词,保持资料和返回数量不变。
query = "回来多久交单"
for label, text in [("原始", query), ("改写", query.replace("交单", "报销"))]:
ids = [h["document_id"] for h in store.search("alpha", text, 3)]
print(label, ids)执行 python seed.py 恢复资料,再执行 python compare_queries.py,结果为:
原始 []
改写 ['travel']观察旧问题有没有变差、新问题有没有改善。如果只对一个问题硬编码答案,其他问法仍会失败;查询改写、向量召回和重排都应该通过同一组保留问题比较。
课后练习展开习题收起习题
练习:让一次故障被测试抓到
临时注释 storage.py 中删除旧片段的代码,运行测试。观察哪个测试报告失败、失败原因是什么,再恢复代码重跑。
参考分析
同一团队的chunk_id要求唯一。更新时若没有先删除旧片段,再插入相同编号会触发约束错误;相关更新测试应失败。恢复后再次运行全部测试,确认故障没有遗留。
测试工具的用法见 pytest入门文档。