Golden Set、真实样本与版本

mkdir l7-golden
cd l7-golden

"""冻结退款评测集，分别比较开发集和留出集的规则版本。"""

import hashlib
import json


# 全部样本均为预置合成数据；同一问题家族不能跨开发集和留出集。
DATASET = {
    "version": "refund-golden-v1",
    "policy": "收到货后0至7天可退款，缺失或非法天数转人工",
    "cases": [
        {"id": "g1", "group": "普通退货", "split": "dev", "days": 3, "expected": "approve"},
        {"id": "g2", "group": "最后一天", "split": "dev", "days": 7, "expected": "approve"},
        {"id": "g3", "group": "超过期限", "split": "dev", "days": 8, "expected": "reject"},
        {"id": "g4", "group": "缺失日期", "split": "dev", "days": None, "expected": "manual"},
        {"id": "h1", "group": "期限内另一日期", "split": "holdout", "days": 6, "expected": "approve"},
        {"id": "h2", "group": "非法日期", "split": "holdout", "days": -1, "expected": "manual"},
    ],
}


def validate(dataset):
    """检查重复编号、标签和问题家族泄漏，防止错误语料混进分数。"""
    if not dataset.get("version") or not dataset.get("cases"):
        raise ValueError("版本和样本不能为空")
    ids, group_splits = set(), {}
    for case in dataset["cases"]:
        if case["id"] in ids:
            raise ValueError("样本编号重复")
        ids.add(case["id"])
        if case["split"] not in ("dev", "holdout"):
            raise ValueError("未知数据分区")
        if case["expected"] not in ("approve", "reject", "manual"):
            raise ValueError("未知参考标签")
        previous = group_splits.setdefault(case["group"], case["split"])
        if previous != case["split"]:
            raise ValueError("同一问题家族跨分区")
    if {case["split"] for case in dataset["cases"]} != {"dev", "holdout"}:
        raise ValueError("需要开发集和留出集")


def fingerprint(dataset):
    """稳定编码后计算摘要；内容或规则变化都会改变这份语料的身份。"""
    validate(dataset)
    encoded = json.dumps(dataset, ensure_ascii=False, sort_keys=True).encode("utf-8")
    return hashlib.sha256(encoded).hexdigest()[:12]


def decide(days, limit):
    """布尔值也属于Python整数家族，使用type避免把True当作第1天。"""
    if type(days) is not int or days < 0:
        return "manual"
    return "approve" if days <= limit else "reject"


def evaluate(dataset, split, limit):
    """只比较指定分区；失败编号便于回到样本，而不只看平均值。"""
    validate(dataset)
    cases = [case for case in dataset["cases"] if case["split"] == split]
    failed = [case["id"] for case in cases
              if decide(case["days"], limit) != case["expected"]]
    return len(cases) - len(failed), len(cases), failed


def main():
    print(f"语料={DATASET['version']} 摘要={fingerprint(DATASET)}")
    # baseline的6天上限故意漏掉第7天；candidate修正为7天。
    for name, limit in (("baseline", 6), ("candidate", 7)):
        for split in ("dev", "holdout"):
            passed, total, failed = evaluate(DATASET, split, limit)
            print(f"{name}/{split}: {passed}/{total}={passed / total:.0%} "
                  f"失败={','.join(failed) or '无'}")


if __name__ == "__main__":
    main()

ls
python -X utf8 golden_lab.py

语料=refund-golden-v1 摘要=ffb5cc87cf7b
baseline/dev: 3/4=75% 失败=g2
baseline/holdout: 2/2=100% 失败=无
candidate/dev: 4/4=100% 失败=无
candidate/holdout: 2/2=100% 失败=无

ls
python -X utf8 -m unittest -v test_golden_lab.py