LLM-as-judge 偏差与校准

mkdir l7-judge
cd l7-judge

"""用固定裁判分数演示阈值校准、误放误拦与位置偏差。"""

import math


# reference是预置参考标签，score是预置裁判输出；程序不调用模型。
CALIBRATION = [
    {"id": "c1", "reference": True, "score": 0.90},
    {"id": "c2", "reference": False, "score": 0.75},
    {"id": "c3", "reference": False, "score": 0.65},
    {"id": "c4", "reference": True, "score": 0.55},
    {"id": "c5", "reference": False, "score": 0.20},
]
HOLDOUT = [
    {"id": "h1", "reference": True, "score": 0.85},
    {"id": "h2", "reference": False, "score": 0.70},
    {"id": "h3", "reference": True, "score": 0.60},
    {"id": "h4", "reference": False, "score": 0.40},
]


def confusion(rows, threshold):
    """大于等于阈值即接受；误放=FP，误拦=FN，正确接受=TP。"""
    if type(threshold) not in (int, float) or not 0 <= threshold <= 1:
        raise ValueError("阈值必须在0到1之间")
    counts = {"TP": 0, "FP": 0, "FN": 0, "TN": 0}
    for row in rows:
        if type(row["reference"]) is not bool:
            raise ValueError("参考标签必须是布尔值")
        score = row["score"]
        if type(score) not in (int, float) or not math.isfinite(score) or not 0 <= score <= 1:
            raise ValueError("分数必须是0到1的有限数值")
        predicted = score >= threshold
        key = ("TP" if predicted else "FN") if row["reference"] else (
            "FP" if predicted else "TN")
        counts[key] += 1
    return counts


def map_choice(order, choice):
    """A/B是显示位置；还原成答案身份后才能比较交换位置前后的结果。"""
    if len(order) != 2 or len(set(order)) != 2 or choice not in ("A", "B"):
        raise ValueError("需要两个不同答案以及A/B选择")
    return order[0 if choice == "A" else 1]


def position_check(first, swapped):
    """相同答案交换位置后，裁判仍应选中相同答案身份。"""
    if tuple(first["order"]) != tuple(reversed(swapped["order"])):
        raise ValueError("第二次必须交换答案位置")
    left = map_choice(first["order"], first["choice"])
    right = map_choice(swapped["order"], swapped["choice"])
    return left == right


def main():
    for threshold in (0.60, 0.80):
        for name, rows in (("calibration", CALIBRATION), ("holdout", HOLDOUT)):
            counts = confusion(rows, threshold)
            correct = counts["TP"] + counts["TN"]
            print(f"阈值={threshold:.2f}/{name}: TP={counts['TP']} FP={counts['FP']} "
                  f"FN={counts['FN']} TN={counts['TN']} 一致={correct}/{len(rows)}"
                  f"={correct / len(rows):.0%}")
    first = {"order": ["brief", "verbose"], "choice": "A"}
    for name, choice in (("偏爱首位", "A"), ("保持答案", "B")):
        swapped = {"order": ["verbose", "brief"], "choice": choice}
        stable = position_check(first, swapped)
        print(f"{name}: 交换后一致={stable} 处理={'保留' if stable else '人工复核队列'}")


if __name__ == "__main__":
    main()

ls
python -X utf8 judge_lab.py

阈值=0.60/calibration: TP=1 FP=2 FN=1 TN=1 一致=2/5=40%
阈值=0.60/holdout: TP=2 FP=1 FN=0 TN=1 一致=3/4=75%
阈值=0.80/calibration: TP=1 FP=0 FN=1 TN=3 一致=4/5=80%
阈值=0.80/holdout: TP=1 FP=0 FN=1 TN=2 一致=3/4=75%
偏爱首位: 交换后一致=False 处理=人工复核队列
保持答案: 交换后一致=True 处理=保留

ls
python -X utf8 -m unittest -v test_judge_lab.py