2.3 LLM-as-Judge 评估方法

1-2 分：回复质量极差
  - 完全不相关、答非所问
  - 包含严重事实错误
  - 输出无意义文本或乱码

3-4 分：回复质量差
  - 部分相关但遗漏关键信息
  - 存在明显错误
  - 格式和组织混乱

5-6 分：回复质量一般
  - 基本回答了问题
  - 有小错误或遗漏
  - 深度和细节不足

7-8 分：回复质量好
  - 准确、有用、组织良好
  - 覆盖主要方面
  - 表达清晰

9-10 分：回复质量出色
  - 全面、准确、有深度
  - 有独到见解或创意
  - 格式完美、表达流畅

JUDGE_PROMPT_SINGLE = """[System]
Please act as an impartial judge and evaluate the quality of the response
provided by an AI assistant to the user question displayed below. Your
evaluation should consider factors such as the helpfulness, relevance,
accuracy, depth, creativity, and level of detail of the response. Begin
your evaluation by providing a short explanation. Be as objective as
possible. After providing your explanation, you must rate the response
on a scale of 1 to 10 by strictly following this format:
"[[rating]]", for example: "Rating: [[5]]".

[Question]
{question}

[The Start of Assistant's Answer]
{answer}
[The End of Assistant's Answer]"""

JUDGE_PROMPT_PAIRWISE = """[System]
Please act as an impartial judge and evaluate the quality of the responses
provided by two AI assistants to the user question displayed below. You
should choose the assistant that follows the user's instructions and
answers the user's question better. Your evaluation should consider
factors such as the helpfulness, relevance, accuracy, depth, creativity,
and level of detail of the response. Begin your evaluation by comparing
the two responses and provide a short explanation. Avoid any position
biases and ensure that the order in which the responses were presented
does not influence your decision. Do not allow the length of the responses
to influence your evaluation. Do not favor certain names of the assistants.
Be as objective as possible. After providing your explanation, output your
final verdict by strictly following this format: "[[A]]" if assistant A
is better, "[[B]]" if assistant B is better, and "[[C]]" for a tie.

[User Question]
{question}

[The Start of Assistant A's Answer]
{answer_a}
[The End of Assistant A's Answer]

[The Start of Assistant B's Answer]
{answer_b}
[The End of Assistant B's Answer]"""

JUDGE_PROMPT_ZH = """[系统]
请你作为一位公正的评委，评估以下 AI 助手对用户问题的回复质量。
你的评估应考虑以下因素：回复的有用性、相关性、准确性、深度、
创造性和详细程度。请先给出简短的评价说明，然后按照以下格式
给出 1-10 分的评分："[[评分]]"，例如："评分：[[7]]"。

[用户问题]
{question}

[助手回复开始]
{answer}
[助手回复结束]"""

import json
import re
import time
from openai import OpenAI

# ===== 1. 配置评委模型 API =====
# 使用阿里云 Model Studio 的 Qwen3-32B
client = OpenAI(
    api_key="YOUR_API_KEY",
    base_url="https://dashscope.aliyuncs.com/compatible-mode/v1"
)

JUDGE_MODEL = "qwen3-32b"

# ===== 2. 评判函数 =====
def judge_response(question: str, answer: str, judge_model: str = JUDGE_MODEL) -> dict:
    """使用 LLM-as-Judge 对回复打分"""
    prompt = JUDGE_PROMPT_SINGLE.format(question=question, answer=answer)

    response = client.chat.completions.create(
        model=judge_model,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.0,       # 评判时使用低 temperature 保证一致性
        max_tokens=512,
    )

    judge_output = response.choices[0].message.content

    # 提取评分
    match = re.search(r'\[\[(\d+)\]\]', judge_output)
    score = int(match.group(1)) if match else None

    return {
        "question": question,
        "answer": answer,
        "judge_output": judge_output,
        "score": score,
    }

# ===== 3. 批量评估 =====
def evaluate_model(model, tokenizer, test_prompts, judge_model=JUDGE_MODEL):
    """对模型在一组提示上进行 LLM-as-Judge 评估"""
    results = []
    for prompt in test_prompts:
        # 生成回复
        answer = generate_response(model, tokenizer, prompt)

        # 评判
        result = judge_response(prompt, answer, judge_model)
        results.append(result)

        # API 限速
        time.sleep(1)

    # 计算统计
    scores = [r["score"] for r in results if r["score"] is not None]
    avg_score = sum(scores) / len(scores) if scores else 0

    return {
        "results": results,
        "avg_score": avg_score,
        "num_evaluated": len(scores),
    }

# ===== 4. 多模型对比 =====
test_prompts = [
    "用三句话解释什么是量子计算",
    "写一首关于秋天的五言绝句",
    "如何用 Python 实现二分查找？请给出完整代码",
    "比较机器学习和深度学习的区别",
    "小明有 23 个苹果，给了小红 7 个，又买了 15 个，现在有几个？",
    # ... 更多测试提示
]

# 评估结果对比表
print(f"{'模型':<30} {'平均分':<10} {'评估数':<10}")
print("=" * 50)
# for model_name, model_obj in models.items():
#     result = evaluate_model(model_obj, tokenizer, test_prompts)
#     print(f"{model_name:<30} {result['avg_score']:<10.2f} {result['num_evaluated']:<10}")

def pairwise_judge(question: str, answer_a: str, answer_b: str) -> dict:
    """成对比较两个模型的回复"""
    # 第一次评判：A在前，B在后
    prompt_ab = JUDGE_PROMPT_PAIRWISE.format(
        question=question, answer_a=answer_a, answer_b=answer_b
    )
    result_ab = client.chat.completions.create(
        model=JUDGE_MODEL,
        messages=[{"role": "user", "content": prompt_ab}],
        temperature=0.0,
        max_tokens=512,
    )

    # 第二次评判：B在前，A在后（缓解位置偏差）
    prompt_ba = JUDGE_PROMPT_PAIRWISE.format(
        question=question, answer_a=answer_b, answer_b=answer_a
    )
    result_ba = client.chat.completions.create(
        model=JUDGE_MODEL,
        messages=[{"role": "user", "content": prompt_ba}],
        temperature=0.0,
        max_tokens=512,
    )

    # 提取判定结果
    verdict_ab = extract_verdict(result_ab.choices[0].message.content)
    verdict_ba = extract_verdict(result_ba.choices[0].message.content)

    # 综合两次结果
    # 如果两次一致 → 高置信度
    # 如果两次不一致 → 标记为 tie
    return {
        "verdict_ab": verdict_ab,
        "verdict_ba": verdict_ba,
        "consistent": is_consistent(verdict_ab, verdict_ba),
    }

# 测量位置偏差的方法
# 对同一对回复，交换位置后重复评判

position_bias_results = []
for question, answer_a, answer_b in test_pairs:
    # A在前
    v1 = judge(question, answer_a, answer_b)  # 假设选了 A

    # B在前
    v2 = judge(question, answer_b, answer_a)  # 如果仍选 A(现在在后面) → 无偏差
                                                # 如果选了 B(现在在前面) → 位置偏差！

    position_bias_results.append({
        "consistent": v1 == flip(v2),  # True = 无偏差
    })

bias_rate = 1 - sum(r["consistent"] for r in position_bias_results) / len(position_bias_results)
print(f"Position bias rate: {bias_rate:.1%}")