676 lines
22 KiB
Python
676 lines
22 KiB
Python
"""
|
||
Integration tests for Metadata Governance runtime pipeline.
|
||
[AC-IDSMETA-18~20] Test routing -> filtering -> retrieval -> fallback chain.
|
||
|
||
Test Matrix:
|
||
- AC-IDSMETA-18: Intent routing with target KB selection
|
||
- AC-IDSMETA-19: Metadata filter injection in RAG retrieval
|
||
- AC-IDSMETA-20: Fallback strategy with structured reason codes
|
||
"""
|
||
|
||
import pytest
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
from typing import Any
|
||
from dataclasses import dataclass, field
|
||
|
||
|
||
@dataclass
|
||
class MockIntentRule:
|
||
"""Mock IntentRule for testing."""
|
||
id: str
|
||
name: str
|
||
response_type: str
|
||
target_kb_ids: list[str] | None = None
|
||
keywords: list[str] | None = None
|
||
patterns: list[str] | None = None
|
||
priority: int = 0
|
||
is_enabled: bool = True
|
||
fixed_reply: str | None = None
|
||
flow_id: str | None = None
|
||
transfer_message: str | None = None
|
||
|
||
|
||
@dataclass
|
||
class MockIntentMatchResult:
|
||
"""Mock IntentMatchResult for testing."""
|
||
rule: MockIntentRule
|
||
match_type: str
|
||
matched: str
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"rule_id": str(self.rule.id),
|
||
"rule_name": self.rule.name,
|
||
"match_type": self.match_type,
|
||
"matched": self.matched,
|
||
"response_type": self.rule.response_type,
|
||
"target_kb_ids": self.rule.target_kb_ids or [],
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class MockRetrievalHit:
|
||
"""Mock RetrievalHit for testing."""
|
||
text: str
|
||
score: float
|
||
source: str
|
||
metadata: dict[str, Any] = field(default_factory=dict)
|
||
|
||
|
||
@dataclass
|
||
class MockRetrievalResult:
|
||
"""Mock RetrievalResult for testing."""
|
||
hits: list[MockRetrievalHit]
|
||
diagnostics: dict[str, Any] = field(default_factory=dict)
|
||
|
||
@property
|
||
def hit_count(self) -> int:
|
||
return len(self.hits)
|
||
|
||
@property
|
||
def max_score(self) -> float:
|
||
return max((h.score for h in self.hits), default=0.0)
|
||
|
||
@property
|
||
def is_empty(self) -> bool:
|
||
return len(self.hits) == 0
|
||
|
||
|
||
class MockIntentRouter:
|
||
"""Mock IntentRouter for testing."""
|
||
|
||
def match(self, message: str, rules: list[MockIntentRule]) -> MockIntentMatchResult | None:
|
||
message_lower = message.lower()
|
||
sorted_rules = sorted(rules, key=lambda r: r.priority, reverse=True)
|
||
for rule in sorted_rules:
|
||
if not rule.is_enabled:
|
||
continue
|
||
if rule.keywords:
|
||
for keyword in rule.keywords:
|
||
if keyword.lower() in message_lower:
|
||
return MockIntentMatchResult(
|
||
rule=rule,
|
||
match_type="keyword",
|
||
matched=keyword,
|
||
)
|
||
if rule.patterns:
|
||
import re
|
||
for pattern in rule.patterns:
|
||
if re.search(pattern, message, re.IGNORECASE):
|
||
return MockIntentMatchResult(
|
||
rule=rule,
|
||
match_type="regex",
|
||
matched=pattern,
|
||
)
|
||
return None
|
||
|
||
|
||
class MockRetriever:
|
||
"""Mock Retriever with metadata filtering support."""
|
||
|
||
def __init__(self, hits: list[MockRetrievalHit] | None = None):
|
||
self._hits = hits or []
|
||
self._last_filter: dict[str, Any] | None = None
|
||
self._last_target_kb_ids: list[str] | None = None
|
||
|
||
async def retrieve(
|
||
self,
|
||
tenant_id: str,
|
||
query: str,
|
||
target_kb_ids: list[str] | None = None,
|
||
metadata_filter: dict[str, Any] | None = None,
|
||
) -> MockRetrievalResult:
|
||
self._last_filter = metadata_filter
|
||
self._last_target_kb_ids = target_kb_ids
|
||
|
||
filtered_hits = []
|
||
for hit in self._hits:
|
||
if metadata_filter:
|
||
match = True
|
||
for key, value in metadata_filter.items():
|
||
if hit.metadata.get(key) != value:
|
||
match = False
|
||
break
|
||
if not match:
|
||
continue
|
||
if target_kb_ids:
|
||
if hit.metadata.get("kb_id") not in target_kb_ids:
|
||
continue
|
||
filtered_hits.append(hit)
|
||
|
||
return MockRetrievalResult(
|
||
hits=filtered_hits,
|
||
diagnostics={
|
||
"filter_applied": metadata_filter is not None,
|
||
"target_kb_ids": target_kb_ids,
|
||
},
|
||
)
|
||
|
||
|
||
class FallbackStrategy:
|
||
"""
|
||
[AC-IDSMETA-20] Fallback strategy with structured reason codes.
|
||
"""
|
||
|
||
REASON_CODES = {
|
||
"NO_INTENT_MATCH": "intent_not_matched",
|
||
"NO_RETRIEVAL_HITS": "retrieval_empty",
|
||
"LOW_CONFIDENCE": "confidence_below_threshold",
|
||
"KB_UNAVAILABLE": "knowledge_base_unavailable",
|
||
"METADATA_FILTER_TOO_STRICT": "filter_excluded_all",
|
||
}
|
||
|
||
def execute(
|
||
self,
|
||
reason: str,
|
||
fallback_kb_id: str | None = None,
|
||
fallback_message: str | None = None,
|
||
) -> dict[str, Any]:
|
||
reason_code = self.REASON_CODES.get(reason, "unknown")
|
||
|
||
result = {
|
||
"fallback_triggered": True,
|
||
"reason_code": reason_code,
|
||
"fallback_type": None,
|
||
"fallback_content": None,
|
||
}
|
||
|
||
if fallback_kb_id:
|
||
result["fallback_type"] = "kb"
|
||
result["fallback_kb_id"] = fallback_kb_id
|
||
elif fallback_message:
|
||
result["fallback_type"] = "fixed"
|
||
result["fallback_content"] = fallback_message
|
||
else:
|
||
result["fallback_type"] = "default"
|
||
result["fallback_content"] = "抱歉，我暂时无法回答您的问题，请稍后重试或联系人工客服。"
|
||
|
||
return result
|
||
|
||
|
||
class TestIntentRouting:
|
||
"""
|
||
[AC-IDSMETA-18] Test intent routing with target KB selection.
|
||
"""
|
||
|
||
def setup_method(self):
|
||
self.router = MockIntentRouter()
|
||
|
||
def test_keyword_match_routes_to_rag(self):
|
||
"""Intent with response_type=rag should route to RAG with target KBs."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="退货咨询",
|
||
response_type="rag",
|
||
target_kb_ids=["kb-return", "kb-policy"],
|
||
keywords=["退货", "退款"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
result = self.router.match("我想退货怎么办", rules)
|
||
|
||
assert result is not None
|
||
assert result.rule.response_type == "rag"
|
||
assert result.rule.target_kb_ids == ["kb-return", "kb-policy"]
|
||
assert result.match_type == "keyword"
|
||
|
||
def test_regex_match_routes_to_flow(self):
|
||
"""Intent with response_type=flow should start script flow."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-2",
|
||
name="订单查询",
|
||
response_type="flow",
|
||
flow_id="flow-order-query",
|
||
patterns=[r"订单.*查询", r"查询.*订单"],
|
||
priority=5,
|
||
)
|
||
]
|
||
|
||
result = self.router.match("帮我查询订单状态", rules)
|
||
|
||
assert result is not None
|
||
assert result.rule.response_type == "flow"
|
||
assert result.rule.flow_id == "flow-order-query"
|
||
assert result.match_type == "regex"
|
||
|
||
def test_fixed_reply_intent(self):
|
||
"""Intent with response_type=fixed should return fixed reply."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-3",
|
||
name="问候",
|
||
response_type="fixed",
|
||
fixed_reply="您好，请问有什么可以帮您？",
|
||
keywords=["你好", "您好"],
|
||
priority=1,
|
||
)
|
||
]
|
||
|
||
result = self.router.match("你好", rules)
|
||
|
||
assert result is not None
|
||
assert result.rule.response_type == "fixed"
|
||
assert result.rule.fixed_reply == "您好，请问有什么可以帮您？"
|
||
|
||
def test_transfer_intent(self):
|
||
"""Intent with response_type=transfer should trigger transfer."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-4",
|
||
name="人工服务",
|
||
response_type="transfer",
|
||
transfer_message="正在为您转接人工客服...",
|
||
keywords=["人工", "转人工"],
|
||
priority=100,
|
||
)
|
||
]
|
||
|
||
result = self.router.match("我要转人工", rules)
|
||
|
||
assert result is not None
|
||
assert result.rule.response_type == "transfer"
|
||
assert result.rule.transfer_message == "正在为您转接人工客服..."
|
||
|
||
def test_priority_ordering(self):
|
||
"""Higher priority rules should be matched first."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-low",
|
||
name="通用问候",
|
||
response_type="fixed",
|
||
fixed_reply="通用问候回复",
|
||
keywords=["你好"],
|
||
priority=1,
|
||
),
|
||
MockIntentRule(
|
||
id="rule-high",
|
||
name="VIP问候",
|
||
response_type="fixed",
|
||
fixed_reply="VIP问候回复",
|
||
keywords=["你好"],
|
||
priority=10,
|
||
),
|
||
]
|
||
|
||
result = self.router.match("你好", rules)
|
||
|
||
assert result is not None
|
||
assert result.rule.id == "rule-high"
|
||
assert result.rule.fixed_reply == "VIP问候回复"
|
||
|
||
def test_no_match_returns_none(self):
|
||
"""No matching intent should return None."""
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="退货",
|
||
response_type="rag",
|
||
keywords=["退货"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
result = self.router.match("今天天气怎么样", rules)
|
||
|
||
assert result is None
|
||
|
||
|
||
class TestMetadataFilterInjection:
|
||
"""
|
||
[AC-IDSMETA-19] Test metadata filter injection in RAG retrieval.
|
||
"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_filter_injection_with_grade_subject_scene(self):
|
||
"""RAG retrieval should inject grade/subject/scene metadata filters."""
|
||
retriever = MockRetriever(hits=[
|
||
MockRetrievalHit(
|
||
text="初一数学知识点",
|
||
score=0.9,
|
||
source="kb",
|
||
metadata={"grade": "初一", "subject": "数学", "scene": "课后辅导", "kb_id": "kb-1"},
|
||
),
|
||
MockRetrievalHit(
|
||
text="初二物理知识点",
|
||
score=0.85,
|
||
source="kb",
|
||
metadata={"grade": "初二", "subject": "物理", "scene": "课后辅导", "kb_id": "kb-1"},
|
||
),
|
||
])
|
||
|
||
metadata_filter = {
|
||
"grade": "初一",
|
||
"subject": "数学",
|
||
"scene": "课后辅导",
|
||
}
|
||
|
||
result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query="数学知识点",
|
||
metadata_filter=metadata_filter,
|
||
)
|
||
|
||
assert retriever._last_filter == metadata_filter
|
||
assert result.hit_count == 1
|
||
assert result.hits[0].metadata["grade"] == "初一"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_target_kb_ids_filtering(self):
|
||
"""RAG retrieval should filter by target KB IDs from intent."""
|
||
retriever = MockRetriever(hits=[
|
||
MockRetrievalHit(
|
||
text="退货政策",
|
||
score=0.9,
|
||
source="kb",
|
||
metadata={"kb_id": "kb-return"},
|
||
),
|
||
MockRetrievalHit(
|
||
text="产品介绍",
|
||
score=0.85,
|
||
source="kb",
|
||
metadata={"kb_id": "kb-product"},
|
||
),
|
||
])
|
||
|
||
result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query="退货",
|
||
target_kb_ids=["kb-return"],
|
||
)
|
||
|
||
assert retriever._last_target_kb_ids == ["kb-return"]
|
||
assert result.hit_count == 1
|
||
assert result.hits[0].metadata["kb_id"] == "kb-return"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_combined_filters(self):
|
||
"""RAG retrieval should combine target KB and metadata filters."""
|
||
retriever = MockRetriever(hits=[
|
||
MockRetrievalHit(
|
||
text="初一数学教材",
|
||
score=0.9,
|
||
source="kb",
|
||
metadata={"grade": "初一", "subject": "数学", "kb_id": "kb-edu"},
|
||
),
|
||
MockRetrievalHit(
|
||
text="初二数学教材",
|
||
score=0.85,
|
||
source="kb",
|
||
metadata={"grade": "初二", "subject": "数学", "kb_id": "kb-edu"},
|
||
),
|
||
MockRetrievalHit(
|
||
text="初一数学练习",
|
||
score=0.8,
|
||
source="kb",
|
||
metadata={"grade": "初一", "subject": "数学", "kb_id": "kb-exercise"},
|
||
),
|
||
])
|
||
|
||
result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query="数学",
|
||
target_kb_ids=["kb-edu"],
|
||
metadata_filter={"grade": "初一"},
|
||
)
|
||
|
||
assert result.hit_count == 1
|
||
assert result.hits[0].metadata["grade"] == "初一"
|
||
assert result.hits[0].metadata["kb_id"] == "kb-edu"
|
||
|
||
|
||
class TestFallbackStrategy:
|
||
"""
|
||
[AC-IDSMETA-20] Test fallback strategy with structured reason codes.
|
||
"""
|
||
|
||
def setup_method(self):
|
||
self.fallback = FallbackStrategy()
|
||
|
||
def test_no_intent_match_fallback(self):
|
||
"""No intent match should trigger fallback with reason code."""
|
||
result = self.fallback.execute(
|
||
reason="NO_INTENT_MATCH",
|
||
fallback_message="抱歉，我不太理解您的问题，请换种方式描述。",
|
||
)
|
||
|
||
assert result["fallback_triggered"] is True
|
||
assert result["reason_code"] == "intent_not_matched"
|
||
assert result["fallback_type"] == "fixed"
|
||
assert "不太理解" in result["fallback_content"]
|
||
|
||
def test_no_retrieval_hits_fallback(self):
|
||
"""No retrieval hits should trigger fallback with reason code."""
|
||
result = self.fallback.execute(
|
||
reason="NO_RETRIEVAL_HITS",
|
||
fallback_kb_id="kb-general",
|
||
)
|
||
|
||
assert result["fallback_triggered"] is True
|
||
assert result["reason_code"] == "retrieval_empty"
|
||
assert result["fallback_type"] == "kb"
|
||
assert result["fallback_kb_id"] == "kb-general"
|
||
|
||
def test_low_confidence_fallback(self):
|
||
"""Low confidence should trigger fallback with reason code."""
|
||
result = self.fallback.execute(
|
||
reason="LOW_CONFIDENCE",
|
||
fallback_message="我对这个回答不太确定，建议您咨询人工客服。",
|
||
)
|
||
|
||
assert result["fallback_triggered"] is True
|
||
assert result["reason_code"] == "confidence_below_threshold"
|
||
assert result["fallback_type"] == "fixed"
|
||
|
||
def test_metadata_filter_too_strict_fallback(self):
|
||
"""Too strict metadata filter should trigger fallback."""
|
||
result = self.fallback.execute(
|
||
reason="METADATA_FILTER_TOO_STRICT",
|
||
fallback_message="没有找到符合条件的答案，请尝试调整筛选条件。",
|
||
)
|
||
|
||
assert result["fallback_triggered"] is True
|
||
assert result["reason_code"] == "filter_excluded_all"
|
||
|
||
def test_default_fallback(self):
|
||
"""Default fallback should be used when no specific fallback provided."""
|
||
result = self.fallback.execute(reason="NO_RETRIEVAL_HITS")
|
||
|
||
assert result["fallback_triggered"] is True
|
||
assert result["fallback_type"] == "default"
|
||
assert "人工客服" in result["fallback_content"]
|
||
|
||
|
||
class TestRoutingFilterRetrievalFallbackChain:
|
||
"""
|
||
[AC-IDSMETA-18, AC-IDSMETA-19, AC-IDSMETA-20] Test complete chain.
|
||
"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_chain_with_intent_match_and_retrieval(self):
|
||
"""Full chain: intent match -> metadata filter -> retrieval -> response."""
|
||
router = MockIntentRouter()
|
||
retriever = MockRetriever(hits=[
|
||
MockRetrievalHit(
|
||
text="退货需在7天内，商品未拆封",
|
||
score=0.9,
|
||
source="kb",
|
||
metadata={"kb_id": "kb-return"},
|
||
),
|
||
])
|
||
fallback = FallbackStrategy()
|
||
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="退货",
|
||
response_type="rag",
|
||
target_kb_ids=["kb-return"],
|
||
keywords=["退货"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
user_message = "我想退货"
|
||
intent_result = router.match(user_message, rules)
|
||
|
||
assert intent_result is not None
|
||
assert intent_result.rule.response_type == "rag"
|
||
|
||
retrieval_result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query=user_message,
|
||
target_kb_ids=intent_result.rule.target_kb_ids,
|
||
)
|
||
|
||
assert retrieval_result.hit_count > 0
|
||
assert not retrieval_result.is_empty
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_chain_no_intent_match_fallback(self):
|
||
"""Full chain: no intent match -> fallback."""
|
||
router = MockIntentRouter()
|
||
fallback = FallbackStrategy()
|
||
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="退货",
|
||
response_type="rag",
|
||
keywords=["退货"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
user_message = "今天天气怎么样"
|
||
intent_result = router.match(user_message, rules)
|
||
|
||
assert intent_result is None
|
||
|
||
fallback_result = fallback.execute(
|
||
reason="NO_INTENT_MATCH",
|
||
fallback_message="抱歉，我无法回答这个问题。",
|
||
)
|
||
|
||
assert fallback_result["fallback_triggered"] is True
|
||
assert fallback_result["reason_code"] == "intent_not_matched"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_chain_no_retrieval_hits_fallback(self):
|
||
"""Full chain: intent match -> no retrieval hits -> fallback."""
|
||
router = MockIntentRouter()
|
||
retriever = MockRetriever(hits=[])
|
||
fallback = FallbackStrategy()
|
||
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="退货",
|
||
response_type="rag",
|
||
target_kb_ids=["kb-return"],
|
||
keywords=["退货"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
user_message = "退货流程是什么"
|
||
intent_result = router.match(user_message, rules)
|
||
|
||
assert intent_result is not None
|
||
|
||
retrieval_result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query=user_message,
|
||
target_kb_ids=intent_result.rule.target_kb_ids,
|
||
)
|
||
|
||
assert retrieval_result.is_empty
|
||
|
||
fallback_result = fallback.execute(
|
||
reason="NO_RETRIEVAL_HITS",
|
||
fallback_kb_id="kb-general",
|
||
)
|
||
|
||
assert fallback_result["fallback_triggered"] is True
|
||
assert fallback_result["reason_code"] == "retrieval_empty"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_chain_with_metadata_filter(self):
|
||
"""Full chain with metadata filter injection."""
|
||
router = MockIntentRouter()
|
||
retriever = MockRetriever(hits=[
|
||
MockRetrievalHit(
|
||
text="初一数学课程大纲",
|
||
score=0.9,
|
||
source="kb",
|
||
metadata={"grade": "初一", "subject": "数学", "scene": "咨询", "kb_id": "kb-edu"},
|
||
),
|
||
MockRetrievalHit(
|
||
text="初二数学课程大纲",
|
||
score=0.85,
|
||
source="kb",
|
||
metadata={"grade": "初二", "subject": "数学", "scene": "咨询", "kb_id": "kb-edu"},
|
||
),
|
||
])
|
||
fallback = FallbackStrategy()
|
||
|
||
rules = [
|
||
MockIntentRule(
|
||
id="rule-1",
|
||
name="课程咨询",
|
||
response_type="rag",
|
||
target_kb_ids=["kb-edu"],
|
||
keywords=["课程", "大纲"],
|
||
priority=10,
|
||
)
|
||
]
|
||
|
||
user_message = "初一数学课程大纲"
|
||
session_metadata = {"grade": "初一", "subject": "数学", "scene": "咨询"}
|
||
|
||
intent_result = router.match(user_message, rules)
|
||
|
||
retrieval_result = await retriever.retrieve(
|
||
tenant_id="tenant-1",
|
||
query=user_message,
|
||
target_kb_ids=intent_result.rule.target_kb_ids if intent_result else None,
|
||
metadata_filter=session_metadata,
|
||
)
|
||
|
||
assert retrieval_result.hit_count == 1
|
||
assert retrieval_result.hits[0].metadata["grade"] == "初一"
|
||
|
||
|
||
class TestReasonCodeStructure:
|
||
"""
|
||
[AC-IDSMETA-20] Test structured reason codes for fallback.
|
||
"""
|
||
|
||
def test_reason_code_format(self):
|
||
"""Reason codes should follow snake_case format."""
|
||
fallback = FallbackStrategy()
|
||
|
||
for reason_key, expected_code in FallbackStrategy.REASON_CODES.items():
|
||
result = fallback.execute(reason=reason_key)
|
||
assert result["reason_code"] == expected_code
|
||
assert "_" in expected_code or expected_code.islower()
|
||
|
||
def test_reason_code_in_diagnostics(self):
|
||
"""Reason code should be included in diagnostics."""
|
||
fallback = FallbackStrategy()
|
||
|
||
result = fallback.execute(reason="NO_RETRIEVAL_HITS")
|
||
|
||
assert "reason_code" in result
|
||
assert result["reason_code"] == "retrieval_empty"
|
||
|
||
def test_unknown_reason_code(self):
|
||
"""Unknown reason should return 'unknown' code."""
|
||
fallback = FallbackStrategy()
|
||
|
||
result = fallback.execute(reason="UNKNOWN_REASON")
|
||
|
||
assert result["reason_code"] == "unknown"
|