Repository navigation
Expand file tree
/
Copy pathrun_eval.py
More file actions
446 lines (374 loc) · 15.4 KB
/
Copy pathrun_eval.py
File metadata and controls
446 lines (374 loc) · 15.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
lite-rag-eval — RAGAS 评测 + Langfuse 追踪
用法:
copy .env.example .env # 填写 AGENT_*、Langfuse、OPENAI_API_KEY
activate.bat # Windows 激活虚拟环境
pip install -r requirements.txt
# 1) 调用智能体 API 收集回答(可先看 results/collected_*.jsonl)
python run_eval.py collect
# 2) 对已有收集结果跑 RAGAS(需配置评判 LLM)
python run_eval.py score --input results/collected_latest.jsonl
# 单条样本诊断(按指标串行,逐步打印日志)
python run_eval.py score-debug --id qa_001
# 3) 一键:收集 + 评分
python run_eval.py all
# 指定样本子集
python run_eval.py collect --ids qa_001,qa_002
"""
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
# 保证从项目根目录直接运行时可导入同级模块
_EVAL_DIR = Path(__file__).resolve().parent
if str(_EVAL_DIR) not in sys.path:
sys.path.insert(0, str(_EVAL_DIR))
from config import EvalConfig
from dataset_loader import GoldenSample, load_golden_dataset
from agent_client import create_agent_client
from langfuse_tracer import LangfuseTracer
from ragas_judge import (
build_ragas_embeddings,
build_ragas_llm,
get_metrics,
preflight_embeddings,
preflight_llm,
preflight_n_generations,
run_preflight,
)
from ragas_prepare import prepare_rows_for_ragas
def _timestamp() -> str:
return datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
def _filter_samples(
samples: list[GoldenSample], ids: str | None
) -> list[GoldenSample]:
if not ids:
return samples
wanted = {x.strip() for x in ids.split(",") if x.strip()}
return [s for s in samples if s.id in wanted]
def cmd_collect(args: argparse.Namespace, config: EvalConfig) -> Path:
samples = load_golden_dataset(config.golden_dataset_path)
samples = _filter_samples(samples, args.ids)
if not samples:
raise SystemExit("没有可评测的样本,请检查 --ids 或 golden_dataset.jsonl")
client = create_agent_client(config)
tracer = LangfuseTracer(config)
config.results_dir.mkdir(parents=True, exist_ok=True)
out_path = config.results_dir / f"collected_{_timestamp()}.jsonl"
latest_path = config.results_dir / "collected_latest.jsonl"
print(f"智能体平台: {config.agent_platform}")
print(f"目标 API: {config.agent_chat_url}")
print(f"样本数: {len(samples)}")
print(f"Langfuse: {'已启用' if tracer.enabled else '未配置(跳过追踪)'}")
rows: list[dict] = []
for i, sample in enumerate(samples, 1):
print(f"[{i}/{len(samples)}] {sample.id}: {sample.question[:40]}...")
with tracer.trace_question(
question_id=sample.id,
question=sample.question,
metadata=sample.metadata,
) as trace:
result = client.chat(sample.question)
row = {
"id": sample.id,
"question": sample.question,
"ground_truth": sample.ground_truth,
"ground_truth_contexts": sample.ground_truth_contexts,
"answer": result.answer,
"contexts": result.contexts,
"chat_id": result.chat_id,
"error": result.error,
"metadata": sample.metadata,
}
if result.error:
print(f" !! 错误: {result.error}")
else:
tracer.log_generation(
trace,
question=sample.question,
answer=result.answer,
contexts=result.contexts,
model=config.agent_platform,
metadata={"sample_id": sample.id, "platform": config.agent_platform},
)
rows.append(row)
tracer.flush()
with out_path.open("w", encoding="utf-8") as f:
for row in rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
latest_path.write_text(out_path.read_text(encoding="utf-8"), encoding="utf-8")
ok = sum(1 for r in rows if not r.get("error"))
print(f"\n完成: 成功 {ok}/{len(rows)}")
print(f"输出: {out_path}")
print(f"最新: {latest_path}")
return out_path
def _load_collected_rows(input_path: Path) -> list[dict]:
rows = []
with input_path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def _filter_rows(rows: list[dict], ids: str | None) -> list[dict]:
if not ids:
return rows
wanted = {x.strip() for x in ids.split(",") if x.strip()}
filtered = [r for r in rows if r.get("id") in wanted]
if not filtered:
raise SystemExit(f"未找到 --ids 指定样本: {ids}")
return filtered
def cmd_score_debug(args: argparse.Namespace, config: EvalConfig) -> None:
if not config.ragas_llm_configured:
raise SystemExit("RAGAS 评判需要 OPENAI_API_KEY,请在 .env 中配置")
input_path = Path(args.input) if args.input else config.results_dir / "collected_latest.jsonl"
if not input_path.exists():
raise SystemExit(f"收集结果不存在: {input_path},请先运行 collect")
rows = _load_collected_rows(input_path)
from ragas_debug import load_row_by_id, score_one_debug
row = load_row_by_id(rows, args.id)
score_one_debug(row, config)
def cmd_score(args: argparse.Namespace, config: EvalConfig) -> None:
if not config.ragas_llm_configured:
raise SystemExit(
"RAGAS 评判需要 OPENAI_API_KEY(或兼容接口),请在 .env 中配置"
)
input_path = Path(args.input) if args.input else config.results_dir / "collected_latest.jsonl"
if not input_path.exists():
raise SystemExit(f"收集结果不存在: {input_path},请先运行 collect")
rows = _load_collected_rows(input_path)
rows = _filter_rows(rows, getattr(args, "ids", None))
eval_rows = [r for r in rows if not r.get("error")]
if not eval_rows:
raise SystemExit("没有成功的样本可供 RAGAS 评分")
eval_rows, trim_stats = prepare_rows_for_ragas(eval_rows, config)
print(
f"RAGAS 输入裁剪: 每样本最多 {config.ragas_max_contexts} 条 context,"
f"answer≤{config.ragas_max_answer_chars} 字,"
f"评判 max_tokens={config.ragas_max_tokens}"
)
if eval_rows:
s0 = trim_stats["samples"][0]
print(
f" 示例 {s0['id']}: context {s0['context_chars_before']}→{s0['context_chars_after']} 字,"
f"answer {s0['answer_chars_before']}→{s0['answer_chars_after']} 字"
)
from datasets import Dataset
from ragas import evaluate
if not config.ragas_skip_preflight:
preflight_llm(config)
preflight_embeddings(config)
preflight_n_generations(config)
evaluator_llm = build_ragas_llm(config)
embeddings = build_ragas_embeddings(config)
# RAGAS 标准字段
dataset_dict = {
"question": [r["question"] for r in eval_rows],
"answer": [r["answer"] for r in eval_rows],
"contexts": [r.get("contexts") or [] for r in eval_rows],
"ground_truth": [r.get("ground_truth") or "" for r in eval_rows],
}
# 有金标准检索片段时启用 context_recall
has_gt_ctx = any(r.get("ground_truth_contexts") for r in eval_rows)
if has_gt_ctx:
dataset_dict["ground_truth_contexts"] = [
r.get("ground_truth_contexts") or [] for r in eval_rows
]
ds = Dataset.from_dict(dataset_dict)
metrics = get_metrics(config, has_gt_ctx=has_gt_ctx)
needs_embedding = any(
m.name in ("answer_relevancy", "context_precision", "context_recall")
for m in metrics
)
if needs_embedding and not config.ragas_embedding_configured:
raise SystemExit(
"当前指标包含 answer_relevancy / context_*,必须配置 Embedding 专用接口:\n"
" RAGAS_EMBEDDING_API_KEY\n"
" RAGAS_EMBEDDING_API_BASE\n"
" RAGAS_EMBEDDING_MODEL\n"
"(可与 OPENAI_API_* 评判 LLM 分离;未填 EMBEDDING_KEY 时回退 OPENAI_API_KEY)"
)
if needs_embedding and embeddings is None:
raise SystemExit("RAGAS_EMBEDDING_MODEL 未配置或 Embedding 客户端创建失败。")
print(f"RAGAS 评判模型: {config.ragas_llm_model}")
print(f"RAGAS 评判 LLM API: {config.openai_api_base}")
if needs_embedding:
print(
f"RAGAS Embedding API: {config.ragas_embedding_api_base_resolved} "
f"model={config.ragas_embedding_model}"
)
print(
f"answer_relevancy strictness={config.ragas_answer_relevancy_strictness} "
f"(对应 API n={config.ragas_answer_relevancy_strictness})"
)
print(f"评测样本: {len(eval_rows)},指标: {[m.name for m in metrics]}")
ctx_chars = [sum(len(c) for c in (r.get("contexts") or [])) for r in eval_rows]
ctx_counts = [len(r.get("contexts") or []) for r in eval_rows]
if ctx_chars:
avg_ctx = sum(ctx_chars) / len(ctx_chars)
max_ctx = max(ctx_chars)
avg_n = sum(ctx_counts) / len(ctx_counts)
print(
f"裁剪后 context: 平均 {avg_n:.1f} 条 / {avg_ctx:.0f} 字,"
f"最大 {max(ctx_counts)} 条 / {max_ctx} 字"
)
if avg_ctx > 4000 or avg_n > 6:
print(
" 提示: context 偏大时 context_precision / faithfulness 会很慢。"
"可减小 .env 中 RAGAS_MAX_CONTEXTS、RAGAS_MAX_CONTEXT_TOTAL_CHARS。"
)
from ragas.run_config import RunConfig
run_config = RunConfig(
timeout=config.ragas_run_timeout,
max_workers=config.ragas_max_workers,
max_retries=config.ragas_max_retries,
)
print(
f"RAGAS 并发: max_workers={config.ragas_max_workers},"
f"单次 LLM 超时={config.ragas_run_timeout}s"
)
eval_kwargs: dict = {
"metrics": metrics,
"llm": evaluator_llm,
"run_config": run_config,
"raise_exceptions": True,
"show_progress": False,
}
if embeddings is not None:
eval_kwargs["embeddings"] = embeddings
from ragas_progress import sample_progress_bar
with sample_progress_bar(len(eval_rows), len(metrics)):
result = evaluate(ds, **eval_kwargs)
df = result.to_pandas()
config.results_dir.mkdir(parents=True, exist_ok=True)
scores_path = config.results_dir / f"ragas_scores_{_timestamp()}.csv"
summary_path = config.results_dir / f"ragas_summary_{_timestamp()}.json"
df.to_csv(scores_path, index=False, encoding="utf-8-sig")
# RAGAS DataFrame 列名为 user_input/response 等,不能仅用 dataset_dict 差集
metric_cols = [m.name for m in metrics if m.name in df.columns]
_RAGAS_DATA_COLS = frozenset(
{
"question",
"answer",
"contexts",
"ground_truth",
"ground_truth_contexts",
"user_input",
"retrieved_contexts",
"response",
"reference",
}
)
if not metric_cols:
metric_cols = [
c
for c in df.columns
if c not in _RAGAS_DATA_COLS and df[c].dtype.kind in "fib"
]
summary = {
"run_name": config.eval_run_name,
"input_file": str(input_path),
"sample_count": len(eval_rows),
"trim_stats": trim_stats,
"metrics": {},
}
import math
for col in metric_cols:
if df[col].dtype.kind in "fi":
series = df[col].dropna()
if series.empty:
summary["metrics"][col] = {
"mean": None,
"min": None,
"max": None,
"valid_count": 0,
"note": "全部为 NaN,多为 LLM/Embedding API 调用失败,请先运行 preflight",
}
else:
summary["metrics"][col] = {
"mean": float(series.mean()),
"min": float(series.min()),
"max": float(series.max()),
"valid_count": int(len(series)),
}
summary_path.write_text(
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
)
# Langfuse 回写分数
tracer = LangfuseTracer(config)
if tracer.enabled:
for i, row in enumerate(eval_rows):
sample_scores = {}
for col in metric_cols:
if i < len(df) and col in df.columns:
val = df.iloc[i][col]
if val != val: # NaN
continue
try:
sample_scores[col] = float(val)
except (TypeError, ValueError):
continue
with tracer.trace_question(
question_id=row["id"],
question=row["question"],
metadata={"phase": "ragas_score"},
) as trace:
tracer.log_ragas_scores(trace, question_id=row["id"], scores=sample_scores)
tracer.flush()
print("\n=== RAGAS 汇总(均值)===")
for name, stats in summary["metrics"].items():
mean = stats.get("mean")
if mean is None or (isinstance(mean, float) and math.isnan(mean)):
print(f" {name}: NaN(有效样本 {stats.get('valid_count', 0)})")
else:
print(f" {name}: {mean:.4f} (n={stats.get('valid_count', '?')})")
print(f"\n明细 CSV: {scores_path}")
print(f"汇总 JSON: {summary_path}")
def cmd_all(args: argparse.Namespace, config: EvalConfig) -> None:
out = cmd_collect(args, config)
args.input = str(out)
cmd_score(args, config)
def main() -> None:
parser = argparse.ArgumentParser(
description="lite-rag-eval:RAG 智能体 RAGAS + Langfuse 评测"
)
sub = parser.add_subparsers(dest="command", required=True)
p_collect = sub.add_parser("collect", help="调用智能体 API 收集回答")
p_collect.add_argument("--ids", help="逗号分隔的样本 id,如 qa_001,qa_002")
p_score = sub.add_parser("score", help="对收集结果运行 RAGAS")
p_score.add_argument(
"--input",
help="collected jsonl 路径,默认 results/collected_latest.jsonl",
)
p_score.add_argument("--ids", help="逗号分隔的样本 id,如 qa_001")
p_score_debug = sub.add_parser(
"score-debug", help="单条样本 RAGAS 诊断(串行、逐步日志)"
)
p_score_debug.add_argument(
"--id", required=True, help="样本 id,如 qa_001"
)
p_score_debug.add_argument(
"--input",
help="collected jsonl 路径,默认 results/collected_latest.jsonl",
)
p_all = sub.add_parser("all", help="collect + score")
p_all.add_argument("--ids", help="逗号分隔的样本 id")
sub.add_parser("preflight", help="预检 RAGAS 评判 LLM / Embedding API")
args = parser.parse_args()
config = EvalConfig.from_env()
if args.command == "collect":
cmd_collect(args, config)
elif args.command == "score":
cmd_score(args, config)
elif args.command == "score-debug":
cmd_score_debug(args, config)
elif args.command == "all":
cmd_all(args, config)
elif args.command == "preflight":
raise SystemExit(run_preflight(config))
if __name__ == "__main__":
main()