多模态质量与实时体验评测

New-Item -ItemType Directory -Force l9-multimodal-eval
cd l9-multimodal-eval

mkdir -p l9-multimodal-eval
cd l9-multimodal-eval

LATENCY_LIMIT = 500
# 固定人工标注材料：应当提醒、预测提醒、判定完成耗时毫秒、输入模态。
# 提醒与不提醒都产出一个判定；时效统一记录判定完成，不依赖是否播放声音。
SAMPLES = [
    (True, True, 300, "图像"), (True, False, 700, "图像"),
    (False, True, 400, "音频"), (False, False, 450, "音频"),
]

def evaluate(samples, limit):
    if limit < 0:
        raise ValueError("延迟门限不能为负数")
    if not samples:
        return {"matrix": [0, 0, 0, 0], "accuracy": None, "timely": None}
    tp = fn = fp = tn = timely = 0
    for truth, prediction, latency, modality in samples:
        if type(truth) is not bool or type(prediction) is not bool or latency < 0:
            raise ValueError("标注必须为布尔值，耗时不能为负数")
        tp += truth and prediction
        fn += truth and not prediction
        fp += not truth and prediction
        tn += not truth and not prediction
        timely += latency <= limit
    # 矩阵顺序：命中、漏报、误报、正确不报；及时率单独计算。
    count = len(samples)
    return {"matrix": [tp, fn, fp, tn],
            "accuracy": (tp + tn) / count, "timely": timely / count}

if __name__ == "__main__":
    result = evaluate(SAMPLES, LATENCY_LIMIT)
    print("混淆矩阵[命中,漏报,误报,正确不报]：" + str(result["matrix"]))
    print("正确率：" + format(result["accuracy"], ".0%"))
    print("判定及时率：" + format(result["timely"], ".0%"))
    for modality in ("图像", "音频"):
        subset = [row for row in SAMPLES if row[3] == modality]
        print(modality + "正确率：" + format(evaluate(subset, LATENCY_LIMIT)["accuracy"], ".0%"))

import unittest
import eval_lab as lab


class MechanismTests(unittest.TestCase):
    # 正常、边界和失败输入都由这里提供，直接运行即可。

    def test_normal(self):
        result = lab.evaluate(lab.SAMPLES, 500)
        self.assertEqual(result["matrix"], [1, 1, 1, 1])
        self.assertEqual(result["timely"], 0.75)

    def test_boundary_empty_and_deadline(self):
        self.assertIsNone(lab.evaluate([], 500)["accuracy"])
        self.assertEqual(lab.evaluate([(True, True, 500, "音频")], 500)["timely"], 1)
        # 不提醒也有判定完成时刻，边界耗时应计入同一分母。
        self.assertEqual(lab.evaluate([(False, False, 500, "图像")], 500)["timely"], 1)
        self.assertEqual(lab.evaluate([(True, False, 501, "图像")], 500)["timely"], 0)

    def test_failure_bad_label(self):
        with self.assertRaises(ValueError):
            lab.evaluate([("yes", True, 3, "音频")], 500)


if __name__ == "__main__":
    unittest.main()

# 在刚才创建的练习目录检查文件，再运行程序。
ls
python -X utf8 eval_lab.py
python -X utf8 -m unittest -v test_eval_lab.py

混淆矩阵[命中,漏报,误报,正确不报]：[1, 1, 1, 1]
正确率：50%
判定及时率：75%
图像正确率：50%
音频正确率：50%

python -X utf8 eval_lab.py