feat(eval): 干净 A/B（生成一次→复核关/开）+ README 为小模型而设计的鲁棒性

README.md

@@ -41,6 +41,20 @@ BanGD 的设计目标就是补上这两点：**识别根因 → 拒绝低级解
 
 ---
 
+## 为低成本 / 小模型而设计的鲁棒性
+
+BanGD 不假设你一定用最强的模型。整条流水线是**按「即使跑在便宜的小模型上也不出问题」来设计的**——小模型的几个典型弱点（格式幻觉、过度报警、措辞漂移），都由代码层吸收掉，而不是靠换更贵的模型来回避：
+
+- **格式幻觉 → 强制结构化 + 校验 + 自动重生成。** 用 Anthropic 的 `tool_use` 强制模型按 JSON Schema 产出，输出在解析边界经 Zod 运行时校验；偶发的不合法输出会**自动重生成一次**。已在 DeepSeek 的兼容端点上验证 `tool_use` 可用，无需 JSON 兜底。
+- **彻底解析失败 → 优雅降级，绝不让 CI 硬挂。** 万一每次生成都无法解析，BanGD 会贴一条**可重试**的评论并记录原始输出供诊断，而不是让评审 job 失败。
+- **过度报警 → 对抗式复核。** 小模型更容易「无中生有」。每条 finding 经多个**反驳者**独立对抗式复核，达到多数（默认全票）判为误报即丢弃——先把小模型的噪声压下去，再呈现给人。
+- **措辞漂移 → 确定性去重。** 去重 key 由代码按 `文件 + 问题类型` 计算、**绝不取自模型措辞**，所以同一问题在多次评审里措辞变了也不会重复刷屏。
+- **成本 → prompt caching + 渐进式披露。** 大块固定内容（系统提示词 / rubric / 范例）打缓存，每个 PR 只有 diff 这条尾巴未缓存；且只把相关维度的 rubric 发给模型。便宜模型也跑得起、跑得快。
+
+> 一句话：**模型越强，架构洞察越深；但模型再便宜，也不会让 BanGD 崩溃、刷屏或被误报淹没。** 默认用 Claude Opus 拿最深的推理；换 DeepSeek 等兼容端点做低成本验证或私有化部署时，上面这些机制保证它依然稳。
+
+---
+
 ## 配置指南（5 分钟接入）
 
 下面以"在 **BanDB 仓库**里启用 BanGD"为例。整个过程只需两步：**加一个 API Key Secret** + **加一个 workflow 文件**。
@@ -114,7 +128,7 @@ DeepSeek 提供 Anthropic 兼容端点。把第 2 步里那两行注释取消即
           model: deepseek-chat
 ```
 
-> 注意：DeepSeek 较弱，**用来验证"链路通不通"，而非"点评准不准"**。生产仍建议用 Claude Opus。
+> 注意：DeepSeek 等小模型在架构推理的**深度**上不及 Opus，更适合验证链路 / 低成本 / 私有化场景。但**跑得稳不稳不取决于模型大小**——[为低成本 / 小模型而设计的鲁棒性](#为低成本--小模型而设计的鲁棒性)那一节的机制（强制结构化 + 重生成、优雅降级、对抗式复核、确定性去重）保证小模型也不会崩溃、刷屏或被误报淹没。追求最深洞察时仍建议 Opus。
 
 ---
 

src/eval/run.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from 'vitest';
+import { runCleanAB, type EvalLlm } from './run.js';
+import type { LlmRequest } from '../core/ports.js';
+import type { PromptTexts } from '../core/prompt.js';
+import type { TokenUsage } from '../core/usage.js';
+import { ALL_DIMENSION_IDS, type DimensionId } from '../core/dimensions.js';
+import { RELATED_PLAN_SCHEMA } from '../core/related.js';
+import { VERDICT_SCHEMA } from '../core/verify.js';
+import { reviewResultJsonSchema } from '../core/schema.js';
+import type { EvalCase } from './corpus.js';
+
+const prompts: PromptTexts = {
+  systemPrompt: 'SYSTEM',
+  rubric: Object.fromEntries(ALL_DIMENSION_IDS.map((id) => [id, `RUBRIC_${id}`])) as Record<DimensionId, string>,
+  examples: { concurrency: 'EX' },
+  generalExample: 'GEN',
+};
+
+// `sync.` keyword → heuristic router picks concurrency, no LLM routing call.
+const DIFF = '+++ b/cache/block.go\n+\tc.hits++ // 读路径无 sync.Mutex 保护';
+
+const evalCase: EvalCase = {
+  id: 'fixture',
+  source: 'NeverENG/BanDB#1',
+  title: 't',
+  body: 'b',
+  diff: DIFF,
+  expected: { findings: [{ file: 'cache/block.go', types: ['并发'] }], generalFindings: [] },
+  groundTruth: 'fixture',
+};
+
+// Generation returns one true finding (cache/block.go) + one false positive (other.go).
+const GENERATED = {
+  changeSummary: 's',
+  overallRisk: '高',
+  findings: [
+    { file: 'cache/block.go', line: 1, severity: '阻塞', type: '并发', rootCause: 'r', whyLowEffortInsufficient: 'w', architecturalSolution: 'a', tradeoffs: 't' },
+    { file: 'other.go', line: 1, severity: '建议', type: '并发', rootCause: 'r', whyLowEffortInsufficient: 'w', architecturalSolution: 'a', tradeoffs: 't' },
+  ],
+  generalFindings: [],
+};
+
+function fakeLlm(usage: TokenUsage, generate: (r: LlmRequest) => unknown): EvalLlm {
+  return {
+    usage,
+    generateStructured: (r: LlmRequest) => {
+      if (r.outputSchema === RELATED_PLAN_SCHEMA) return Promise.resolve({ paths: [] });
+      return Promise.resolve(generate(r));
+    },
+  };
+}
+
+describe('runCleanAB (clean A/B over one generation)', () => {
+  it('generates ONCE and scores both configs off that single generation', async () => {
+    let genCalls = 0;
+    const genLlm = fakeLlm({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) => {
+      if (r.outputSchema === reviewResultJsonSchema) genCalls++;
+      return GENERATED;
+    });
+    // Refute only the false-positive finding (other.go); keep cache/block.go.
+    const verifyLlm = fakeLlm({ inputTokens: 20, outputTokens: 5, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) => {
+      if (r.outputSchema === VERDICT_SCHEMA) return { refuted: r.user.includes('other.go'), reason: 'x' };
+      throw new Error('verify client should only get verdict calls');
+    });
+
+    const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm);
+
+    // The whole point: a single generation, reused — not regenerated per config.
+    expect(genCalls).toBe(1);
+
+    // Baseline = raw findings (TP + FP). Optimized = same batch, FP refuted away.
+    expect(baseline.cases[0]?.predFindings).toHaveLength(2);
+    expect(optimized.cases[0]?.predFindings).toEqual([{ file: 'cache/block.go', type: '并发' }]);
+
+    // Verification lifts precision on the SAME generation (1/2 → 1/1), recall unchanged.
+    expect(baseline.findings.precision).toBeCloseTo(0.5);
+    expect(optimized.findings.precision).toBeCloseTo(1);
+    expect(baseline.findings.recall).toBeCloseTo(1);
+    expect(optimized.findings.recall).toBeCloseTo(1);
+  });
+
+  it('attributes tokens cleanly: baseline = generation, optimized = generation + verification', async () => {
+    const genLlm = fakeLlm({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => GENERATED);
+    const verifyLlm = fakeLlm({ inputTokens: 20, outputTokens: 5, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) =>
+      r.outputSchema === VERDICT_SCHEMA ? { refuted: false, reason: 'x' } : null,
+    );
+
+    const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm);
+
+    expect(baseline.tokens).toBe(110); // generation only
+    expect(optimized.tokens).toBe(135); // 110 generation (shared) + 25 verification
+  });
+
+  it('records a per-case error without crashing the run, for both configs', async () => {
+    const genLlm = fakeLlm({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => ({
+      // malformed: fails Zod validation in review() → InvalidModelOutputError after retries
+      nonsense: true,
+    }));
+    const verifyLlm = fakeLlm({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => ({ refuted: false, reason: 'x' }));
+
+    const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm);
+
+    expect(baseline.cases[0]?.error).toBeTruthy();
+    expect(optimized.cases[0]?.error).toBeTruthy();
+    expect(baseline.cases[0]?.predFindings).toHaveLength(0);
+  });
+});

src/eval/run.ts

@@ -16,12 +16,14 @@
  */
 import { readFileSync, existsSync, writeFileSync } from 'node:fs';
 import { dirname, join, parse } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 import { review } from '../core/review.js';
-import type { PrContext } from '../core/ports.js';
+import { verifyFindings, verifyGeneralFindings } from '../core/verify.js';
+import type { LlmClient, PrContext } from '../core/ports.js';
+import type { PromptTexts } from '../core/prompt.js';
 import { AnthropicLlmClient } from '../shell/llm.js';
 import { loadPromptTexts } from '../shell/prompts.js';
-import { formatUsage, totalTokens } from '../core/usage.js';
+import { formatUsage, totalTokens, type TokenUsage } from '../core/usage.js';
 import { loadCorpus, type EvalCase } from './corpus.js';
 import {
   scoreFindings,
@@ -38,10 +40,13 @@ interface Config {
   verifyVotes: number;
 }
 
-const CONFIGS: Config[] = [
-  { label: 'baseline（无对抗式复核）', verifyVotes: 0 },
-  { label: '优化后（对抗式复核 3 票/条）', verifyVotes: 3 },
-];
+/** Adversarial refuters per finding in the "optimized" config (DESIGN §六.3). */
+const VERIFY_VOTES = 3;
+const BASELINE_LABEL = 'baseline（无对抗式复核）';
+const OPTIMIZED_LABEL = `优化后（对抗式复核 ${VERIFY_VOTES} 票/条）`;
+
+/** An LLM client that also exposes accumulated token usage (the shell client). */
+export type EvalLlm = LlmClient & { usage: TokenUsage };
 
 function resolveKey(): string {
   const env = process.env['ANTHROPIC_API_KEY'] ?? process.env['DEEPSEEK_API_KEY'];
@@ -78,54 +83,101 @@ interface ConfigResult {
   usageText: string;
 }
 
-async function runConfig(
-  config: Config,
+function scoreCase(
+  c: EvalCase,
+  predFindings: PredictedFinding[],
+  predGeneral: PredictedGeneral[],
+): CaseResult {
+  const findingMetrics = scoreFindings(c.expected.findings, predFindings);
+  const generalFp = scoreGeneral(c.expected.generalFindings, predGeneral).fp;
+  return { case: c, predFindings, predGeneral, findingMetrics, generalFp };
+}
+
+/**
+ * Clean A/B (the key fix vs the first eval). Generate findings ONCE per PR, then
+ * score TWO configs off that single generation: baseline = the raw findings,
+ * optimized = the SAME findings after adversarial verification. The only variable
+ * between configs is verification, so the precision delta can no longer be
+ * confounded by the model's generation randomness — which the old design (two
+ * independent `review()` calls, one per config) could not separate.
+ *
+ * Generation and verification use separate clients so token cost splits cleanly:
+ * baseline pays for generation, optimized pays for the SAME generation plus the
+ * verification overhead. Injecting both clients (rather than constructing them)
+ * keeps this unit testable with fakes.
+ */
+export async function runCleanAB(
   cases: EvalCase[],
-  apiKey: string,
-  baseURL: string | undefined,
-  model: string | undefined,
-): Promise<ConfigResult> {
-  // Fresh client per config → clean per-config token totals.
-  const llm = new AnthropicLlmClient({
-    apiKey,
-    ...(model ? { model } : {}),
-    ...(baseURL ? { baseURL } : {}),
-  });
-  const prompts = await loadPromptTexts();
-  const results: CaseResult[] = [];
+  prompts: PromptTexts,
+  genLlm: EvalLlm,
+  verifyLlm: EvalLlm,
+): Promise<{ baseline: ConfigResult; optimized: ConfigResult }> {
+  const baselineCases: CaseResult[] = [];
+  const optimizedCases: CaseResult[] = [];
 
   for (const c of cases) {
     const pr: PrContext = {
       metadata: { title: c.title, body: c.body, number: null },
       getDiff: () => Promise.resolve(c.diff),
-      readFile: () => Promise.resolve(null), // diff-only context, uniform across configs
+      readFile: () => Promise.resolve(null), // diff-only context, shared by both configs
     };
     try {
-      const { result } = await review({ llm, pr }, prompts, { verifyVotes: config.verifyVotes });
-      const predFindings = result.findings.map((f) => ({ file: f.file, type: f.type }));
-      const predGeneral = result.generalFindings.map((g) => ({ file: g.file, category: g.category }));
-      const findingMetrics = scoreFindings(c.expected.findings, predFindings);
-      const generalFp = scoreGeneral(c.expected.generalFindings, predGeneral).fp;
-      results.push({ case: c, predFindings, predGeneral, findingMetrics, generalFp });
+      // Generate ONCE (verification off) — this single generation feeds both configs.
+      const { result } = await review({ llm: genLlm, pr }, prompts, { verifyVotes: 0 });
+      const base = scoreCase(
+        c,
+        result.findings.map((f) => ({ file: f.file, type: f.type })),
+        result.generalFindings.map((g) => ({ file: g.file, category: g.category })),
+      );
+      baselineCases.push(base);
+
+      // Verify the SAME findings — the single A/B variable.
+      const ctx = { changeSummary: result.changeSummary, diff: c.diff };
+      const [arch, general] = await Promise.all([
+        verifyFindings(verifyLlm, result.findings, ctx, VERIFY_VOTES),
+        verifyGeneralFindings(verifyLlm, result.generalFindings, ctx, VERIFY_VOTES),
+      ]);
+      const opt = scoreCase(
+        c,
+        arch.kept.map((f) => ({ file: f.file, type: f.type })),
+        general.kept.map((g) => ({ file: g.file, category: g.category })),
+      );
+      optimizedCases.push(opt);
+
       console.log(
-        `  [${config.label}] ${c.source}: 架构 finding ${predFindings.length} 条 / 普通 ${predGeneral.length} 条 ` +
-          `(P=${pct(findingMetrics.precision)} R=${pct(findingMetrics.recall)})`,
+        `  ${c.source}: 生成架构 ${base.predFindings.length} / 普通 ${base.predGeneral.length} 条；` +
+          `复核后架构 ${opt.predFindings.length} / 普通 ${opt.predGeneral.length} 条`,
       );
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
-      const empty = scoreFindings(c.expected.findings, []);
-      results.push({ case: c, predFindings: [], predGeneral: [], findingMetrics: empty, generalFp: 0, error: msg });
-      console.log(`  [${config.label}] ${c.source}: ERROR ${msg}`);
+      const errResult: CaseResult = { ...scoreCase(c, [], []), error: msg };
+      baselineCases.push(errResult);
+      optimizedCases.push(errResult);
+      console.log(`  ${c.source}: ERROR ${msg}`);
     }
   }
 
+  const genTokens = totalTokens(genLlm.usage);
+  const verifyTokens = totalTokens(verifyLlm.usage);
+
   return {
-    config,
-    cases: results,
-    findings: aggregate(results.map((r) => r.findingMetrics)),
-    generalFp: results.reduce((s, r) => s + r.generalFp, 0),
-    tokens: totalTokens(llm.usage),
-    usageText: formatUsage(llm.usage),
+    baseline: {
+      config: { label: BASELINE_LABEL, verifyVotes: 0 },
+      cases: baselineCases,
+      findings: aggregate(baselineCases.map((r) => r.findingMetrics)),
+      generalFp: baselineCases.reduce((s, r) => s + r.generalFp, 0),
+      tokens: genTokens,
+      usageText: formatUsage(genLlm.usage),
+    },
+    optimized: {
+      config: { label: OPTIMIZED_LABEL, verifyVotes: VERIFY_VOTES },
+      cases: optimizedCases,
+      findings: aggregate(optimizedCases.map((r) => r.findingMetrics)),
+      generalFp: optimizedCases.reduce((s, r) => s + r.generalFp, 0),
+      // Generation is shared with baseline; optimized additionally pays for verification.
+      tokens: genTokens + verifyTokens,
+      usageText: `生成 ${formatUsage(genLlm.usage)}；复核 ${formatUsage(verifyLlm.usage)}`,
+    },
   };
 }
 
@@ -151,10 +203,14 @@ function reportMarkdown(results: ConfigResult[], cases: EvalCase[], meta: { date
       '只认结构锚点、不比对措辞，避免被表述方式钻空子。',
   );
   L.push(
-    '- **单变量 A/B**：两次运行只差一个变量——**对抗式复核关 / 开（0 票 vs 3 票）**；' +
-      '模型、提示词（含 few-shot）、上下文全部一致，每个配置用全新客户端以隔离 token 计数。',
+    `- **干净 A/B（生成一次 → 复核关/开）**：每个 PR 只**生成一次** finding，再对**同一批** finding 分别评分——` +
+      `baseline=不复核，优化=对同一批做 ${VERIFY_VOTES} 票对抗式复核。唯一变量就是复核本身，` +
+      '彻底消除了「生成随机性」对精确率结论的混淆（旧设计两次独立生成 finding，无法区分精确率变化来自复核、还是来自这次生成本就多报/少报）。',
+  );
+  L.push(
+    '- **token 拆分**：生成与复核用两个客户端，干净归因——baseline=生成开销，优化=生成开销 + 复核开销；' +
+      '二者的**生成部分是同一次调用**，完全可比。上下文两配置一致：仅 diff（`readFile` 返回 null）。',
   );
-  L.push('- **上下文**：仅 diff（`readFile` 返回 null），两配置一致，保证 A/B 可比。');
   L.push('');
   L.push('## 语料');
   L.push('');
@@ -242,11 +298,15 @@ async function main(): Promise<void> {
   const corpus = await loadCorpus();
   console.log(`语料：${corpus.cases.length} 个真实 PR（${corpus.repo}）`);
 
-  const results: ConfigResult[] = [];
-  for (const config of CONFIGS) {
-    console.log(`\n=== 运行配置：${config.label} ===`);
-    results.push(await runConfig(config, corpus.cases, apiKey, baseURL, model));
-  }
+  // Two fresh clients: one for the single generation, one for verification — so
+  // token cost splits cleanly between the configs (see runCleanAB).
+  const mkClient = (): EvalLlm =>
+    new AnthropicLlmClient({ apiKey, ...(model ? { model } : {}), ...(baseURL ? { baseURL } : {}) });
+  const prompts = await loadPromptTexts();
+
+  console.log('\n=== 生成一次 → 对同一批做 复核关/开（干净 A/B）===');
+  const { baseline, optimized } = await runCleanAB(corpus.cases, prompts, mkClient(), mkClient());
+  const results: ConfigResult[] = [baseline, optimized];
 
   const date = new Date().toISOString().slice(0, 10);
   const md = reportMarkdown(results, corpus.cases, {
@@ -264,7 +324,13 @@ async function main(): Promise<void> {
   console.log(`\n报告已写入 ${out}`);
 }
 
-main().catch((e: unknown) => {
-  console.error(e instanceof Error ? e.message : String(e));
-  process.exit(1);
-});
+// Run only when invoked as the entry (node build/eval/run.js), NOT when imported
+// by a test — importing must not kick off a live eval / exit the process.
+const invokedDirectly =
+  process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (invokedDirectly) {
+  main().catch((e: unknown) => {
+    console.error(e instanceof Error ? e.message : String(e));
+    process.exit(1);
+  });
+}