在构建企业级 RAG（检索增强生成）系统时，幻觉（Hallucination）问题一直是困扰开发者的核心挑战。当 LLM 基于检索到的文档生成回答时，可能会捏造不存在的信息、歪曲原文意图，或将多个文档的知识错误融合。本文将深入探讨如何使用 RAGAS 和 TruLens 这两大主流框架实现幻觉检测与消除，并提供可执行的完整代码示例。
平台对比：HolySheheep AI vs 官方 API vs 其他 Relay 服务
| Vergleichskriterium | HolySheep AI | Offizielle API (OpenAI) | Andere Relay-Dienste |
|---|---|---|---|
| GPT-4.1 Preis | $8/MTok (¥1=$1) | $60/MTok | $15-25/MTok |
| Claude Sonnet 4.5 | $15/MTok | $18/MTok | $16-20/MTok |
| Gemini 2.5 Flash | $2.50/MTok | $2.50/MTok | $3-5/MTok |
| DeepSeek V3.2 | $0.42/MTok | N/A | $0.50-1/MTok |
| Latenz | <50ms | 100-500ms | 80-300ms |
| Zahlungsmethoden | WeChat/Alipay/USD | Nur Kreditkarte | Variiert |
| Kostenlose Credits | ✓ Ja | ✗ Nein | Selten |
| Kostenersparnis | 85%+ | 0% | 20-60% |
作为长期使用多个 API 提供商的开发者，我强烈推荐 HolySheep AI 作为 RAG 评估管道的核心 LLM 调用源。其亚 50 毫秒的延迟和极具竞争力的价格使大规模幻觉检测在经济上完全可行。
什么是 RAG 幻觉？
RAG 幻觉是指大语言模型在生成回答时出现的以下几种典型错误：
- 事实捏造：生成文档中不存在的数据、数字或事件
- 引用错误：声称某观点来自特定文档，但原文档并无此内容
- 上下文混淆：将多个不相关文档的信息错误组合
- 过度推断：从文档的有限信息中得出超出范围的结论
核心工具介绍
RAGAS（Retrieval Augmented Generation Assessment）
RAGAS 是一个专门为 RAG 管道设计的评估框架，提供以下关键指标：
- Faithfulness：回答对检索文档的事实忠诚度
- Answer Relevancy：回答与问题的相关性
- Context Precision：检索上下文的精确度
- Context Recall：检索上下文对回答的召回率
TruLens
TruLens 提供了更细粒度的反馈机制：
- Groundedness：回答的事实依据评分
- Context Relevance：上下文与问题的语义相关性
- RAG Triadic Evaluation：输入-上下文-输出三联评估
实战环境搭建
安装依赖
# 创建虚拟环境
python -m venv rag_evaluation
source rag_evaluation/bin/activate # Windows: rag_evaluation\Scripts\activate
安装核心依赖
pip install ragas langchain-openai langchain-community
pip install trulens-eval sqlalchemy
pip install tiktoken openai faiss-cpu
pip install pandas numpy python-dotenv配置 HolySheep AI API
import os
from dotenv import load_dotenv
.env 文件配置
HOLYSHEEP_API_KEY=your_actual_key_here
load_dotenv()
HolySheep AI 配置 - 核心配置
HOLYSHEHEP_CONFIG = {
"base_url": "https://api.holysheep.ai/v1",
"api_key": os.getenv("HOLYSHEEP_API_KEY", "YOUR_HOLYSHEEP_API_KEY"),
"model": "gpt-4.1", # 或 claude-sonnet-4.5, gemini-2.5-flash, deepseek-v3.2
"temperature": 0.3,
"max_tokens": 2000
}
验证连接
from openai import OpenAI
client = OpenAI(
base_url=HOLYSHEHEP_CONFIG["base_url"],
api_key=HOLYSHEHEP_CONFIG["api_key"]
)
简单连接测试
try:
response = client.chat.completions.create(
model=HOLYSHEHEP_CONFIG["model"],
messages=[{"role": "user", "content": "Hello, test connection"}],
max_tokens=10
)
print(f"✓ API 连接成功！响应: {response.choices[0].message.content}")
except Exception as e:
print(f"✗ 连接失败: {e}")完整 RAG 幻觉检测系统实现
"""
RAG 幻觉检测与消除系统
使用 RAGAS + TruLens + HolySheep AI
"""
import json
import warnings
from dataclasses import dataclass, field
from typing import List, Dict, Optional, Any
from datetime import datetime
import pandas as pd
RAGAS 相关
from ragas import evaluate
from ragas.metrics import (
faithfulness,
answer_relevancy,
context_precision,
context_recall,
response_quality
)
from ragas.dataset_schema import SingleTurnSample
TruLens 相关
from trulens_eval import TruChain, Feedback, Select
from trulens_eval.feedback import Groundedness, ContextRelevance
from trulens_eval.feedback.provider.openai import OpenAI as TruOpenAI
from trulens_eval.app import App
LangChain 相关
from langchain_openai import ChatOpenAI
from langchain_community.vectorstores import FAISS
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from langchain_core.documents import Document
from langchain_core.prompts import ChatPromptTemplate
warnings.filterwarnings("ignore")
@dataclass
class RAGHaluinationResult:
"""幻觉检测结果数据结构"""
question: str
context: List[str]
response: str
ragas_scores: Dict[str, float] = field(default_factory=dict)
trulens_scores: Dict[str, float] = field(default_factory=dict)
is_hallucination: bool = False
hallucination_type: Optional[str] = None
detected_at: str = field(default_factory=lambda: datetime.now().isoformat())
class HolySheepLLMFactory:
"""HolySheep AI LLM 工厂类"""
def __init__(self, api_key: str, base_url: str = "https://api.holysheep.ai/v1"):
self.api_key = api_key
self.base_url = base_url
def get_chat_model(self, model: str = "gpt-4.1", temperature: float = 0.3):
"""获取 ChatOpenAI 实例（用于 RAGAS）"""
return ChatOpenAI(
model=model,
api_key=self.api_key,
base_url=self.base_url,
temperature=temperature
)
def get_trulens_provider(self):
"""获取 TruLens 的 OpenAI 反馈提供者"""
return TruOpenAI(
api_key=self.api_key,
base_url=self.base_url
)
class RAGHallucinationDetector:
"""RAG 幻觉检测器"""
def __init__(
self,
api_key: str,
model: str = "gpt-4.1",
base_url: str = "https://api.holysheep.ai/v1"
):
self.llm_factory = HolySheepLLMFactory(api_key, base_url)
self.chat_model = self.llm_factory.get_chat_model(model)
self.embeddings = OpenAIEmbeddings(
api_key=api_key,
base_url=base_url
)
self.vectorstore = None
self.retriever = None
self._init_trulens()
def _init_trulens(self):
"""初始化 TruLens 反馈函数"""
self.trulens_provider = self.llm_factory.get_trulens_provider()
# 基于 groundedness 的反馈（衡量回答是否基于给定上下文）
self.groundedness = Feedback(
self.trulens_provider.groundedness_measure,
name="Groundedness"
).on(
Select.Record response
).on(
Select.Record context
).aggregate(self.trulens_provider.groundedness_aggregation)
# 上下文相关性反馈
self.context_relevance = Feedback(
self.trulens_provider.context_relevance,
name="Context Relevance"
).on(
Select.Record question
).on(
Select.Record context
).aggregate(self.trulens_provider.relevance_aggregation)
def setup_vectorstore(self, documents: List[str], metadatas: List[Dict] = None):
"""构建向量数据库"""
docs = [
Document(page_content=doc, metadata=metadatas[i] if metadatas else {})
for i, doc in enumerate(documents)
]
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=500,
chunk_overlap=50
)
split_docs = text_splitter.split_documents(docs)
self.vectorstore = FAISS.from_documents(split_docs, self.embeddings)
self.retriever = self.vectorstore.as_retriever(
search_kwargs={"k": 3}
)
print(f"✓ 向量数据库已构建，包含 {len(split_docs)} 个文档块")
def retrieve_context(self, question: str) -> List[str]:
"""检索相关上下文"""
if not self.retriever:
raise ValueError("请先调用 setup_vectorstore() 方法")
docs = self.retriever.invoke(question)
return [doc.page_content for doc in docs]
def generate_response(self, question: str, context: List[str]) -> str:
"""使用 LLM 生成回答"""
context_str = "\n\n".join([f"[文档 {i+1}]: {ctx}" for i, ctx in enumerate(context)])
prompt = f"""基于以下参考文档回答问题。如果文档中没有相关信息，请明确说明"文档中未提供此信息"。
参考文档：
{context_str}
问题：{question}
回答（仅基于上述文档）："""
response = self.chat_model.invoke(prompt)
return response.content if hasattr(response, 'content') else str(response)
def detect_with_ragas(
self,
question: str,
answer: str,
context: List[str],
contexts: List[List[str]]
) -> Dict[str, float]:
"""使用 RAGAS 进行评估"""
# 构建 RAGAS 数据集格式
eval_data = [
SingleTurnSample(
user_input=question,
response=answer,
reference_contexts=contexts[0] if contexts else context
)
]
try:
# 执行评估
result = evaluate(
eval_data,
metrics=[
faithfulness,
answer_relevancy,
context_precision,
context_recall
],
llm=self.chat_model,
embeddings=self.embeddings
)
scores = {
"faithfulness": float(result["faithfulness"].iloc[0]),
"answer_relevancy": float(result["answer_relevancy"].iloc[0]),
"context_precision": float(result["context_precision"].iloc[0]),
"context_recall": float(result["context_recall"].iloc[0])
}
return scores
except Exception as e:
print(f"RAGAS 评估出错: {e}")
return {}
def detect_with_trulens(
self,
question: str,
context: List[str],
response: str
) -> Dict[str, float]:
"""使用 TruLens 进行评估"""
# 构建 TruLens 记录
record = {
"question": question,
"context": context,
"response": response
}
try:
groundedness_score = self.groundedness(record)
relevance_score = self.context_relevance(record)
return {
"trulens_groundedness": groundedness_score,
"trulens_relevance": relevance_score
}
except Exception as e:
print(f"TruLens 评估出错: {e}")
return {}
def analyze_hallucination_type(
self,
question: str,
context: List[str],
response: str,
ragas_scores: Dict,
trulens_scores: Dict
) -> Optional[str]:
"""分析幻觉类型"""
issues = []
# 检测事实捏造
if ragas_scores.get("faithfulness", 1.0) < 0.5:
issues.append("事实捏造")
# 检测引用错误
if trulens_scores.get("trulens_groundedness", 1.0) < 0.6:
issues.append("引用错误")
# 检测上下文混淆
if ragas_scores.get("context_precision", 1.0) < 0.4:
issues.append("上下文混淆")
# 检测过度推断
if ragas_scores.get("answer_relevancy", 1.0) < 0.5:
issues.append("过度推断")
return " + ".join(issues) if issues else None
def full_detection(
self,
question: str,
use_ragas: bool = True,
use_trulens: bool = True
) -> RAGHaluinationResult:
"""完整的幻觉检测流程"""
# 1. 检索上下文
context = self.retrieve_context(question)
# 2. 生成回答
response = self.generate_response(question, context)
# 3. RAGAS 评估
ragas_scores = {}
if use_ragas:
ragas_scores = self.detect_with_ragas(
question, response, context, [context]
)
# 4. TruLens 评估
trulens_scores = {}
if use_trulens:
trulens_scores = self.detect_with_trulens(question, context, response)
# 5. 综合判断
is_hallucination = (
ragas_scores.get("faithfulness", 1.0) < 0.6 or
trulens_scores.get("trulens_groundedness", 1.0) < 0.5
)
hallucination_type = self.analyze_hallucination_type(
question, context, response, ragas_scores, trulens_scores
)
return RAGHaluinationResult(
question=question,
context=context,
response=response,
ragas_scores=ragas_scores,
trulens_scores=trulens_scores,
is_hallucination=is_hallucination,
hallucination_type=hallucination_type
)
============ 使用示例 ============
def main():
# 初始化检测器
detector = RAGHallucinationDetector(
api_key="YOUR_HOLYSHEEP_API_KEY",
model="gpt-4.1",
base_url="https://api.holysheep.ai/v1"
)
# 准备测试文档
documents = [
"苹果公司成立于1976年4月1日，由史蒂夫·乔布斯、史蒂夫·沃兹尼亚克和罗纳德·韦恩共同创立。",
"2023年苹果公司的年收入达到3832亿美元，创历史新高。",
"苹果公司的总部位于加利福尼亚州库比蒂诺。",
"苹果公司于2020年推出自研M1芯片，标志着从Intel处理器的转型。",
"乔布斯于2011年10月5日去世，享年56岁。"
]
# 构建向量数据库
detector.setup_vectorstore(documents)
# 执行幻觉检测
test_questions = [
"苹果公司的总部在哪里？",
"苹果公司是什么时候成立的？",
"苹果公司的CEO是谁？", # 这个问题文档中没有答案，可能触发幻觉
"苹果2023年的营收是多少？"
]
results = []
for question in test_questions:
print(f"\n{'='*60}")
print(f"问题: {question}")
print('='*60)
result = detector.full_detection(question)
print(f"回答: {result.response}")
print(f"\nRAGAS 评分:")
for metric, score in result.ragas_scores.items():
print(f" - {metric}: {score:.3f}")
print(f"\nTruLens 评分:")
for metric, score in result.trulens_scores.items():
print(f" - {metric}: {score:.3f}")
print(f"\n幻觉检测结果:")
print(f" - 是否幻觉: {'⚠️ 是' if result.is_hallucination else '✓ 否'}")
if result.hallucination_type:
print(f" - 幻觉类型: {result.hallucination_type}")
results.append(result)
# 生成报告
report = pd.DataFrame([{
"问题": r.question,
"幻觉": "是" if r.is_hallucination else "否",
"Faithfulness": r.ragas_scores.get("faithfulness", "N/A"),
"Groundedness": r.trulens_scores.get("trulens_groundedness", "N/A"),
"幻觉类型": r.hallucination_type or "无"
} for r in results])
print("\n\n" + "="*60)
print("📊 检测报告摘要")
print("="*60)
print(report.to_string(index=False))
return results
if __name__ == "__main__":
results = main()幻觉消除策略
"""
RAG 幻觉消除策略实现
包含提示词工程、置信度校准、混合检索等技术
"""
from enum import Enum
from typing import Tuple, Optional
import json
class ConfidenceLevel(Enum):
"""置信度级别"""
HIGH = "high"
MEDIUM = "medium"
LOW = "low"
UNCERTAIN = "uncertain"
class HallucinationMitigator:
"""幻觉缓解器"""
def __init__(self, detector, threshold_faithfulness: float = 0.7,
threshold_groundedness: float = 0.6):
self.detector = detector
self.threshold_faithfulness = threshold_faithfulness
self.threshold_groundedness = threshold_groundedness
# 缓解策略提示词
self.safe_prompts = {
"factual": """你是一个严谨的事实核查助手。请根据提供的文档回答问题。
重要规则：
1. 只使用文档中明确包含的信息
2. 如果信息不完整，请说"文档中信息不足"