247 lines
7.9 KiB
Python
247 lines
7.9 KiB
Python
"""
|
||
LLM judge for intent arbitration.
|
||
[AC-AISVC-118, AC-AISVC-119] LLM-based intent arbitration.
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import time
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from app.services.intent.models import (
|
||
FusionConfig,
|
||
LlmJudgeInput,
|
||
LlmJudgeResult,
|
||
RuleMatchResult,
|
||
SemanticMatchResult,
|
||
)
|
||
|
||
if TYPE_CHECKING:
|
||
from app.services.llm.base import LLMClient
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class LlmJudge:
|
||
"""
|
||
[AC-AISVC-118] LLM-based intent arbitrator.
|
||
|
||
Triggered when:
|
||
- Rule vs Semantic conflict
|
||
- Gray zone (low confidence)
|
||
- Multiple intent candidates with similar scores
|
||
"""
|
||
|
||
JUDGE_PROMPT = """你是一个意图识别仲裁器。根据用户消息和候选意图，判断最匹配的意图。
|
||
|
||
用户消息：{message}
|
||
|
||
候选意图：
|
||
{candidates}
|
||
|
||
请返回 JSON 格式（不要包含```json标记）：
|
||
{{
|
||
"intent_id": "最匹配的意图ID",
|
||
"intent_name": "意图名称",
|
||
"confidence": 0.0-1.0之间的置信度,
|
||
"reasoning": "判断理由"
|
||
}}"""
|
||
|
||
def __init__(
|
||
self,
|
||
llm_client: "LLMClient",
|
||
config: FusionConfig,
|
||
):
|
||
"""
|
||
Initialize LLM judge.
|
||
|
||
Args:
|
||
llm_client: LLM client for generating responses
|
||
config: Fusion configuration
|
||
"""
|
||
self._llm_client = llm_client
|
||
self._config = config
|
||
|
||
def should_trigger(
|
||
self,
|
||
rule_result: RuleMatchResult,
|
||
semantic_result: SemanticMatchResult,
|
||
config: FusionConfig | None = None,
|
||
) -> tuple[bool, str]:
|
||
"""
|
||
[AC-AISVC-118] Check if LLM judge should be triggered.
|
||
|
||
Trigger conditions:
|
||
1. Conflict: Rule and Semantic match different intents with close scores
|
||
2. Gray zone: Max confidence in gray zone range
|
||
3. Multi-intent: Multiple candidates with similar scores
|
||
|
||
Args:
|
||
rule_result: Rule matching result
|
||
semantic_result: Semantic matching result
|
||
config: Optional config override
|
||
|
||
Returns:
|
||
Tuple of (should_trigger, trigger_reason)
|
||
"""
|
||
effective_config = config or self._config
|
||
|
||
if not effective_config.llm_judge_enabled:
|
||
return False, "disabled"
|
||
|
||
rule_score = rule_result.score
|
||
semantic_score = semantic_result.top_score
|
||
|
||
if rule_score > 0 and semantic_score > 0:
|
||
if semantic_result.candidates:
|
||
top_semantic_rule_id = semantic_result.candidates[0].rule.id
|
||
if rule_result.rule_id != top_semantic_rule_id:
|
||
if abs(rule_score - semantic_score) < effective_config.conflict_threshold:
|
||
logger.info(
|
||
f"[AC-AISVC-118] LLM judge triggered: rule_semantic_conflict, "
|
||
f"rule_id={rule_result.rule_id}, semantic_id={top_semantic_rule_id}, "
|
||
f"rule_score={rule_score}, semantic_score={semantic_score}"
|
||
)
|
||
return True, "rule_semantic_conflict"
|
||
|
||
max_score = max(rule_score, semantic_score)
|
||
if effective_config.min_trigger_threshold < max_score < effective_config.gray_zone_threshold:
|
||
logger.info(
|
||
f"[AC-AISVC-118] LLM judge triggered: gray_zone, "
|
||
f"max_score={max_score}"
|
||
)
|
||
return True, "gray_zone"
|
||
|
||
if len(semantic_result.candidates) >= 2:
|
||
top1_score = semantic_result.candidates[0].score
|
||
top2_score = semantic_result.candidates[1].score
|
||
if abs(top1_score - top2_score) < effective_config.multi_intent_threshold:
|
||
logger.info(
|
||
f"[AC-AISVC-118] LLM judge triggered: multi_intent, "
|
||
f"top1_score={top1_score}, top2_score={top2_score}"
|
||
)
|
||
return True, "multi_intent"
|
||
|
||
return False, ""
|
||
|
||
async def judge(
|
||
self,
|
||
input_data: LlmJudgeInput,
|
||
tenant_id: str,
|
||
) -> LlmJudgeResult:
|
||
"""
|
||
[AC-AISVC-119] Perform LLM arbitration.
|
||
|
||
Args:
|
||
input_data: Judge input with message and candidates
|
||
tenant_id: Tenant ID for isolation
|
||
|
||
Returns:
|
||
LlmJudgeResult with arbitration decision
|
||
"""
|
||
start_time = time.time()
|
||
|
||
candidates_text = "\n".join([
|
||
f"- ID: {c['id']}, 名称: {c['name']}, 描述: {c.get('description', 'N/A')}"
|
||
for c in input_data.candidates
|
||
])
|
||
|
||
prompt = self.JUDGE_PROMPT.format(
|
||
message=input_data.message,
|
||
candidates=candidates_text,
|
||
)
|
||
|
||
try:
|
||
from app.services.llm.base import LLMConfig
|
||
|
||
response = await asyncio.wait_for(
|
||
self._llm_client.generate(
|
||
messages=[{"role": "user", "content": prompt}],
|
||
config=LLMConfig(
|
||
max_tokens=200,
|
||
temperature=0,
|
||
),
|
||
),
|
||
timeout=self._config.llm_judge_timeout_ms / 1000,
|
||
)
|
||
|
||
result = self._parse_response(response.content or "")
|
||
duration_ms = int((time.time() - start_time) * 1000)
|
||
|
||
tokens_used = 0
|
||
if response.usage:
|
||
tokens_used = response.usage.get("total_tokens", 0)
|
||
|
||
logger.info(
|
||
f"[AC-AISVC-119] LLM judge completed for tenant={tenant_id}, "
|
||
f"intent_id={result.get('intent_id')}, confidence={result.get('confidence', 0):.3f}, "
|
||
f"duration={duration_ms}ms, tokens={tokens_used}"
|
||
)
|
||
|
||
return LlmJudgeResult(
|
||
intent_id=result.get("intent_id"),
|
||
intent_name=result.get("intent_name"),
|
||
score=float(result.get("confidence", 0.5)),
|
||
reasoning=result.get("reasoning"),
|
||
duration_ms=duration_ms,
|
||
tokens_used=tokens_used,
|
||
triggered=True,
|
||
)
|
||
|
||
except asyncio.TimeoutError:
|
||
duration_ms = int((time.time() - start_time) * 1000)
|
||
logger.warning(
|
||
f"[AC-AISVC-119] LLM judge timeout for tenant={tenant_id}, "
|
||
f"timeout={self._config.llm_judge_timeout_ms}ms"
|
||
)
|
||
return LlmJudgeResult(
|
||
intent_id=None,
|
||
intent_name=None,
|
||
score=0.0,
|
||
reasoning="LLM timeout",
|
||
duration_ms=duration_ms,
|
||
tokens_used=0,
|
||
triggered=True,
|
||
)
|
||
except Exception as e:
|
||
duration_ms = int((time.time() - start_time) * 1000)
|
||
logger.error(
|
||
f"[AC-AISVC-119] LLM judge error for tenant={tenant_id}: {e}"
|
||
)
|
||
return LlmJudgeResult(
|
||
intent_id=None,
|
||
intent_name=None,
|
||
score=0.0,
|
||
reasoning=f"LLM error: {str(e)}",
|
||
duration_ms=duration_ms,
|
||
tokens_used=0,
|
||
triggered=True,
|
||
)
|
||
|
||
def _parse_response(self, content: str) -> dict[str, Any]:
|
||
"""
|
||
Parse LLM response to extract JSON result.
|
||
|
||
Args:
|
||
content: LLM response content
|
||
|
||
Returns:
|
||
Parsed dictionary with intent_id, intent_name, confidence, reasoning
|
||
"""
|
||
try:
|
||
cleaned = content.strip()
|
||
if cleaned.startswith("```json"):
|
||
cleaned = cleaned[7:]
|
||
if cleaned.startswith("```"):
|
||
cleaned = cleaned[3:]
|
||
if cleaned.endswith("```"):
|
||
cleaned = cleaned[:-3]
|
||
cleaned = cleaned.strip()
|
||
|
||
result: dict[str, Any] = json.loads(cleaned)
|
||
return result
|
||
except json.JSONDecodeError as e:
|
||
logger.warning(f"[AC-AISVC-119] Failed to parse LLM response: {e}")
|
||
return {}
|