27届大模型面试准备（五十七）：多模态大模型评测与自动化 Benchmark 工程——从能力 taxonomy 到回归守护

                多模态评测工程总架构
+-----------------------------------------------------------------+
|                       模型 (VLM) 制品                           |
+-----------------------------------------------------------------+
            |                            |
   +--------v---------+        +---------v----------+
   |  离线评测 Harness |        |   在线评测 / 影子     |
   |  (Benchmark 自动化)|       |   (A/B + 流量回放)   |
   +--------+---------+        +---------+----------+
            |                            |
   +--------v----------------------------v----------+
   |         能力 Taxonomy 与 指标层                    |
   |  感知/OCR/空间/计数/推理/幻觉/安全/长图/视频        |
   +--------------------------------+-----------------+
            |                            |
   +--------v---------+        +---------v----------+
   |  回归比对 (diff)  |        |  Badcase 挖掘与归因  |
   |  HEAD vs BASE    |        |  聚类 + 人工标注回流  |
   +------------------+        +---------------------+
            |
   +--------v---------+
   |  门禁 Gate (CI)   |
   |  非劣化才放行     |
   +------------------+

# 能力 taxonomy 的最小配置
TAXONOMY = {
    "perception": {"weight": 0.15, "sets": ["coco_cap", "vg"]},
    "ocr":        {"weight": 0.15, "sets": ["ocrbench", "docvqa"]},
    "spatial":    {"weight": 0.10, "sets": ["spatial_eval"]},
    "reasoning":  {"weight": 0.25, "sets": ["mmbench", "mmmu"]},
    "halluc":     {"weight": 0.15, "sets": ["pope", "hallusion"]},
    "safety":     {"weight": 0.10, "sets": ["redteam"]},
    "longimg":    {"weight": 0.05, "sets": ["longimg"]},
    "video":      {"weight": 0.05, "sets": ["mvbench"]},
}

def aggregate(scores: dict) -> float:
    # scores: {dim: acc}
    return sum(TAXONOMY[d]["weight"] * scores[d] for d in TAXONOMY)

import json, concurrent.futures as cf

def run_one(sample, model):
    try:
        ans = model.generate(image=sample["image"],
                             prompt=sample["question"])
        return {"id": sample["id"], "pred": ans,
                "gold": sample["answer"], "ok": True}
    except Exception as e:
        return {"id": sample["id"], "pred": None,
                "gold": sample["answer"], "ok": False, "err": str(e)}

def evaluate(dataset, model, workers=16):
    results = []
    with cf.ThreadPoolExecutor(workers) as ex:
        futs = [ex.submit(run_one, s, model) for s in dataset]
        for f in cf.as_completed(futs):
            results.append(f.result())
    # 失败样本单独存，后续补跑
    failed = [r for r in results if not r["ok"]]
    json.dump(failed, open("failed.jsonl", "w"), ensure_ascii=False)
    return [r for r in results if r["ok"]]

# 幻觉对抗：构造「图中无此物」的反事实问句
counterfactual = [
    "图中有几只斑马？",      # 图里其实是马
    "图片里的红色汽车是什么型号？",  # 图里没有汽车
]
# 期望模型回答「没有/无法确定」，统计误肯定率
def halluc_rate(records):
    wrong = sum(1 for r in records
                if r["gold"] == "无" and "有" in r["pred"])
    return wrong / max(1, len(records))

commit/tag 触发 -> 拉取固定 benchmark 版 -> 推理(集群) ->
判分 -> 写数据库(模型版本, 维度, 分数, 样本级结果) ->
生成雷达图 + diff 报告 -> 非劣化则打标放行，否则钉钉/飞书告警

-- 退化样本定位：BASE vs HEAD 在 perception 维度
SELECT h.sample_id, b.pred AS base_pred, h.pred AS head_pred
FROM results h
JOIN results b ON h.sample_id = b.sample_id
WHERE h.model_version = 'HEAD' AND b.model_version = 'BASE'
  AND h.dataset = 'coco_cap'
  AND b.ok = 1 AND h.ok = 0;   -- HEAD 错而 BASE 对 => 回归

def rag_eval(query, gold_docs, retrieved, answer, judge):
    ret_recall = len(set(gold_docs) & set(retrieved[:5])) / len(gold_docs)
    faith = judge.is_faithful(answer, retrieved)   # 是否只依据检索证据
    return {"retrieval_recall@5": ret_recall, "faithfulness": faith}