diff --git a/README.md b/README.md index 379bb55..37122ed 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,20 @@ BanGD 的设计目标就是补上这两点：**识别根因 → 拒绝低级解 --- +## 为低成本 / 小模型而设计的鲁棒性 + +BanGD 不假设你一定用最强的模型。整条流水线是**按「即使跑在便宜的小模型上也不出问题」来设计的**——小模型的几个典型弱点（格式幻觉、过度报警、措辞漂移），都由代码层吸收掉，而不是靠换更贵的模型来回避： + +- **格式幻觉 → 强制结构化 + 校验 + 自动重生成。** 用 Anthropic 的 `tool_use` 强制模型按 JSON Schema 产出，输出在解析边界经 Zod 运行时校验；偶发的不合法输出会**自动重生成一次**。已在 DeepSeek 的兼容端点上验证 `tool_use` 可用，无需 JSON 兜底。 +- **彻底解析失败 → 优雅降级，绝不让 CI 硬挂。** 万一每次生成都无法解析，BanGD 会贴一条**可重试**的评论并记录原始输出供诊断，而不是让评审 job 失败。 +- **过度报警 → 对抗式复核。** 小模型更容易「无中生有」。每条 finding 经多个**反驳者**独立对抗式复核，达到多数（默认全票）判为误报即丢弃——先把小模型的噪声压下去，再呈现给人。 +- **措辞漂移 → 确定性去重。** 去重 key 由代码按 `文件 + 问题类型` 计算、**绝不取自模型措辞**，所以同一问题在多次评审里措辞变了也不会重复刷屏。 +- **成本 → prompt caching + 渐进式披露。** 大块固定内容（系统提示词 / rubric / 范例）打缓存，每个 PR 只有 diff 这条尾巴未缓存；且只把相关维度的 rubric 发给模型。便宜模型也跑得起、跑得快。 + +> 一句话：**模型越强，架构洞察越深；但模型再便宜，也不会让 BanGD 崩溃、刷屏或被误报淹没。** 默认用 Claude Opus 拿最深的推理；换 DeepSeek 等兼容端点做低成本验证或私有化部署时，上面这些机制保证它依然稳。 + +--- + ## 配置指南（5 分钟接入） 下面以"在 **BanDB 仓库**里启用 BanGD"为例。整个过程只需两步：**加一个 API Key Secret** + **加一个 workflow 文件**。 @@ -114,7 +128,7 @@ DeepSeek 提供 Anthropic 兼容端点。把第 2 步里那两行注释取消即 model: deepseek-chat ``` -> 注意：DeepSeek 较弱，**用来验证"链路通不通"，而非"点评准不准"**。生产仍建议用 Claude Opus。 +> 注意：DeepSeek 等小模型在架构推理的**深度**上不及 Opus，更适合验证链路 / 低成本 / 私有化场景。但**跑得稳不稳不取决于模型大小**——[为低成本 / 小模型而设计的鲁棒性](#为低成本--小模型而设计的鲁棒性)那一节的机制（强制结构化 + 重生成、优雅降级、对抗式复核、确定性去重）保证小模型也不会崩溃、刷屏或被误报淹没。追求最深洞察时仍建议 Opus。 --- diff --git a/src/eval/run.test.ts b/src/eval/run.test.ts new file mode 100644 index 0000000..ca0806b --- /dev/null +++ b/src/eval/run.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect } from 'vitest'; +import { runCleanAB, type EvalLlm } from './run.js'; +import type { LlmRequest } from '../core/ports.js'; +import type { PromptTexts } from '../core/prompt.js'; +import type { TokenUsage } from '../core/usage.js'; +import { ALL_DIMENSION_IDS, type DimensionId } from '../core/dimensions.js'; +import { RELATED_PLAN_SCHEMA } from '../core/related.js'; +import { VERDICT_SCHEMA } from '../core/verify.js'; +import { reviewResultJsonSchema } from '../core/schema.js'; +import type { EvalCase } from './corpus.js'; + +const prompts: PromptTexts = { + systemPrompt: 'SYSTEM', + rubric: Object.fromEntries(ALL_DIMENSION_IDS.map((id) => [id, `RUBRIC_${id}`])) as Record, + examples: { concurrency: 'EX' }, + generalExample: 'GEN', +}; + +// `sync.` keyword → heuristic router picks concurrency, no LLM routing call. +const DIFF = '+++ b/cache/block.go\n+\tc.hits++ // 读路径无 sync.Mutex 保护'; + +const evalCase: EvalCase = { + id: 'fixture', + source: 'NeverENG/BanDB#1', + title: 't', + body: 'b', + diff: DIFF, + expected: { findings: [{ file: 'cache/block.go', types: ['并发'] }], generalFindings: [] }, + groundTruth: 'fixture', +}; + +// Generation returns one true finding (cache/block.go) + one false positive (other.go). +const GENERATED = { + changeSummary: 's', + overallRisk: '高', + findings: [ + { file: 'cache/block.go', line: 1, severity: '阻塞', type: '并发', rootCause: 'r', whyLowEffortInsufficient: 'w', architecturalSolution: 'a', tradeoffs: 't' }, + { file: 'other.go', line: 1, severity: '建议', type: '并发', rootCause: 'r', whyLowEffortInsufficient: 'w', architecturalSolution: 'a', tradeoffs: 't' }, + ], + generalFindings: [], +}; + +function fakeLlm(usage: TokenUsage, generate: (r: LlmRequest) => unknown): EvalLlm { + return { + usage, + generateStructured: (r: LlmRequest) => { + if (r.outputSchema === RELATED_PLAN_SCHEMA) return Promise.resolve({ paths: [] }); + return Promise.resolve(generate(r)); + }, + }; +} + +describe('runCleanAB (clean A/B over one generation)', () => { + it('generates ONCE and scores both configs off that single generation', async () => { + let genCalls = 0; + const genLlm = fakeLlm({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) => { + if (r.outputSchema === reviewResultJsonSchema) genCalls++; + return GENERATED; + }); + // Refute only the false-positive finding (other.go); keep cache/block.go. + const verifyLlm = fakeLlm({ inputTokens: 20, outputTokens: 5, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) => { + if (r.outputSchema === VERDICT_SCHEMA) return { refuted: r.user.includes('other.go'), reason: 'x' }; + throw new Error('verify client should only get verdict calls'); + }); + + const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm); + + // The whole point: a single generation, reused — not regenerated per config. + expect(genCalls).toBe(1); + + // Baseline = raw findings (TP + FP). Optimized = same batch, FP refuted away. + expect(baseline.cases[0]?.predFindings).toHaveLength(2); + expect(optimized.cases[0]?.predFindings).toEqual([{ file: 'cache/block.go', type: '并发' }]); + + // Verification lifts precision on the SAME generation (1/2 → 1/1), recall unchanged. + expect(baseline.findings.precision).toBeCloseTo(0.5); + expect(optimized.findings.precision).toBeCloseTo(1); + expect(baseline.findings.recall).toBeCloseTo(1); + expect(optimized.findings.recall).toBeCloseTo(1); + }); + + it('attributes tokens cleanly: baseline = generation, optimized = generation + verification', async () => { + const genLlm = fakeLlm({ inputTokens: 100, outputTokens: 10, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => GENERATED); + const verifyLlm = fakeLlm({ inputTokens: 20, outputTokens: 5, cacheReadTokens: 0, cacheCreationTokens: 0 }, (r) => + r.outputSchema === VERDICT_SCHEMA ? { refuted: false, reason: 'x' } : null, + ); + + const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm); + + expect(baseline.tokens).toBe(110); // generation only + expect(optimized.tokens).toBe(135); // 110 generation (shared) + 25 verification + }); + + it('records a per-case error without crashing the run, for both configs', async () => { + const genLlm = fakeLlm({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => ({ + // malformed: fails Zod validation in review() → InvalidModelOutputError after retries + nonsense: true, + })); + const verifyLlm = fakeLlm({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }, () => ({ refuted: false, reason: 'x' })); + + const { baseline, optimized } = await runCleanAB([evalCase], prompts, genLlm, verifyLlm); + + expect(baseline.cases[0]?.error).toBeTruthy(); + expect(optimized.cases[0]?.error).toBeTruthy(); + expect(baseline.cases[0]?.predFindings).toHaveLength(0); + }); +}); diff --git a/src/eval/run.ts b/src/eval/run.ts index bcc69f6..6b8ab6d 100644 --- a/src/eval/run.ts +++ b/src/eval/run.ts @@ -16,12 +16,14 @@ */ import { readFileSync, existsSync, writeFileSync } from 'node:fs'; import { dirname, join, parse } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { review } from '../core/review.js'; -import type { PrContext } from '../core/ports.js'; +import { verifyFindings, verifyGeneralFindings } from '../core/verify.js'; +import type { LlmClient, PrContext } from '../core/ports.js'; +import type { PromptTexts } from '../core/prompt.js'; import { AnthropicLlmClient } from '../shell/llm.js'; import { loadPromptTexts } from '../shell/prompts.js'; -import { formatUsage, totalTokens } from '../core/usage.js'; +import { formatUsage, totalTokens, type TokenUsage } from '../core/usage.js'; import { loadCorpus, type EvalCase } from './corpus.js'; import { scoreFindings, @@ -38,10 +40,13 @@ interface Config { verifyVotes: number; } -const CONFIGS: Config[] = [ - { label: 'baseline（无对抗式复核）', verifyVotes: 0 }, - { label: '优化后（对抗式复核 3 票/条）', verifyVotes: 3 }, -]; +/** Adversarial refuters per finding in the "optimized" config (DESIGN §六.3). */ +const VERIFY_VOTES = 3; +const BASELINE_LABEL = 'baseline（无对抗式复核）'; +const OPTIMIZED_LABEL = `优化后（对抗式复核 ${VERIFY_VOTES} 票/条）`; + +/** An LLM client that also exposes accumulated token usage (the shell client). */ +export type EvalLlm = LlmClient & { usage: TokenUsage }; function resolveKey(): string { const env = process.env['ANTHROPIC_API_KEY'] ?? process.env['DEEPSEEK_API_KEY']; @@ -78,54 +83,101 @@ interface ConfigResult { usageText: string; } -async function runConfig( - config: Config, +function scoreCase( + c: EvalCase, + predFindings: PredictedFinding[], + predGeneral: PredictedGeneral[], +): CaseResult { + const findingMetrics = scoreFindings(c.expected.findings, predFindings); + const generalFp = scoreGeneral(c.expected.generalFindings, predGeneral).fp; + return { case: c, predFindings, predGeneral, findingMetrics, generalFp }; +} + +/** + * Clean A/B (the key fix vs the first eval). Generate findings ONCE per PR, then + * score TWO configs off that single generation: baseline = the raw findings, + * optimized = the SAME findings after adversarial verification. The only variable + * between configs is verification, so the precision delta can no longer be + * confounded by the model's generation randomness — which the old design (two + * independent `review()` calls, one per config) could not separate. + * + * Generation and verification use separate clients so token cost splits cleanly: + * baseline pays for generation, optimized pays for the SAME generation plus the + * verification overhead. Injecting both clients (rather than constructing them) + * keeps this unit testable with fakes. + */ +export async function runCleanAB( cases: EvalCase[], - apiKey: string, - baseURL: string | undefined, - model: string | undefined, -): Promise { - // Fresh client per config → clean per-config token totals. - const llm = new AnthropicLlmClient({ - apiKey, - ...(model ? { model } : {}), - ...(baseURL ? { baseURL } : {}), - }); - const prompts = await loadPromptTexts(); - const results: CaseResult[] = []; + prompts: PromptTexts, + genLlm: EvalLlm, + verifyLlm: EvalLlm, +): Promise<{ baseline: ConfigResult; optimized: ConfigResult }> { + const baselineCases: CaseResult[] = []; + const optimizedCases: CaseResult[] = []; for (const c of cases) { const pr: PrContext = { metadata: { title: c.title, body: c.body, number: null }, getDiff: () => Promise.resolve(c.diff), - readFile: () => Promise.resolve(null), // diff-only context, uniform across configs + readFile: () => Promise.resolve(null), // diff-only context, shared by both configs }; try { - const { result } = await review({ llm, pr }, prompts, { verifyVotes: config.verifyVotes }); - const predFindings = result.findings.map((f) => ({ file: f.file, type: f.type })); - const predGeneral = result.generalFindings.map((g) => ({ file: g.file, category: g.category })); - const findingMetrics = scoreFindings(c.expected.findings, predFindings); - const generalFp = scoreGeneral(c.expected.generalFindings, predGeneral).fp; - results.push({ case: c, predFindings, predGeneral, findingMetrics, generalFp }); + // Generate ONCE (verification off) — this single generation feeds both configs. + const { result } = await review({ llm: genLlm, pr }, prompts, { verifyVotes: 0 }); + const base = scoreCase( + c, + result.findings.map((f) => ({ file: f.file, type: f.type })), + result.generalFindings.map((g) => ({ file: g.file, category: g.category })), + ); + baselineCases.push(base); + + // Verify the SAME findings — the single A/B variable. + const ctx = { changeSummary: result.changeSummary, diff: c.diff }; + const [arch, general] = await Promise.all([ + verifyFindings(verifyLlm, result.findings, ctx, VERIFY_VOTES), + verifyGeneralFindings(verifyLlm, result.generalFindings, ctx, VERIFY_VOTES), + ]); + const opt = scoreCase( + c, + arch.kept.map((f) => ({ file: f.file, type: f.type })), + general.kept.map((g) => ({ file: g.file, category: g.category })), + ); + optimizedCases.push(opt); + console.log( - ` [${config.label}] ${c.source}: 架构 finding ${predFindings.length} 条 / 普通 ${predGeneral.length} 条 ` + - `(P=${pct(findingMetrics.precision)} R=${pct(findingMetrics.recall)})`, + ` ${c.source}: 生成架构 ${base.predFindings.length} / 普通 ${base.predGeneral.length} 条；` + + `复核后架构 ${opt.predFindings.length} / 普通 ${opt.predGeneral.length} 条`, ); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - const empty = scoreFindings(c.expected.findings, []); - results.push({ case: c, predFindings: [], predGeneral: [], findingMetrics: empty, generalFp: 0, error: msg }); - console.log(` [${config.label}] ${c.source}: ERROR ${msg}`); + const errResult: CaseResult = { ...scoreCase(c, [], []), error: msg }; + baselineCases.push(errResult); + optimizedCases.push(errResult); + console.log(` ${c.source}: ERROR ${msg}`); } } + const genTokens = totalTokens(genLlm.usage); + const verifyTokens = totalTokens(verifyLlm.usage); + return { - config, - cases: results, - findings: aggregate(results.map((r) => r.findingMetrics)), - generalFp: results.reduce((s, r) => s + r.generalFp, 0), - tokens: totalTokens(llm.usage), - usageText: formatUsage(llm.usage), + baseline: { + config: { label: BASELINE_LABEL, verifyVotes: 0 }, + cases: baselineCases, + findings: aggregate(baselineCases.map((r) => r.findingMetrics)), + generalFp: baselineCases.reduce((s, r) => s + r.generalFp, 0), + tokens: genTokens, + usageText: formatUsage(genLlm.usage), + }, + optimized: { + config: { label: OPTIMIZED_LABEL, verifyVotes: VERIFY_VOTES }, + cases: optimizedCases, + findings: aggregate(optimizedCases.map((r) => r.findingMetrics)), + generalFp: optimizedCases.reduce((s, r) => s + r.generalFp, 0), + // Generation is shared with baseline; optimized additionally pays for verification. + tokens: genTokens + verifyTokens, + usageText: `生成 ${formatUsage(genLlm.usage)}；复核 ${formatUsage(verifyLlm.usage)}`, + }, }; } @@ -151,10 +203,14 @@ function reportMarkdown(results: ConfigResult[], cases: EvalCase[], meta: { date '只认结构锚点、不比对措辞，避免被表述方式钻空子。', ); L.push( - '- **单变量 A/B**：两次运行只差一个变量——**对抗式复核关 / 开（0 票 vs 3 票）**；' + - '模型、提示词（含 few-shot）、上下文全部一致，每个配置用全新客户端以隔离 token 计数。', + `- **干净 A/B（生成一次 → 复核关/开）**：每个 PR 只**生成一次** finding，再对**同一批** finding 分别评分——` + + `baseline=不复核，优化=对同一批做 ${VERIFY_VOTES} 票对抗式复核。唯一变量就是复核本身，` + + '彻底消除了「生成随机性」对精确率结论的混淆（旧设计两次独立生成 finding，无法区分精确率变化来自复核、还是来自这次生成本就多报/少报）。', + ); + L.push( + '- **token 拆分**：生成与复核用两个客户端，干净归因——baseline=生成开销，优化=生成开销 + 复核开销；' + + '二者的**生成部分是同一次调用**，完全可比。上下文两配置一致：仅 diff（`readFile` 返回 null）。', ); - L.push('- **上下文**：仅 diff（`readFile` 返回 null），两配置一致，保证 A/B 可比。'); L.push(''); L.push('## 语料'); L.push(''); @@ -242,11 +298,15 @@ async function main(): Promise { const corpus = await loadCorpus(); console.log(`语料：${corpus.cases.length} 个真实 PR（${corpus.repo}）`); - const results: ConfigResult[] = []; - for (const config of CONFIGS) { - console.log(`\n=== 运行配置：${config.label} ===`); - results.push(await runConfig(config, corpus.cases, apiKey, baseURL, model)); - } + // Two fresh clients: one for the single generation, one for verification — so + // token cost splits cleanly between the configs (see runCleanAB). + const mkClient = (): EvalLlm => + new AnthropicLlmClient({ apiKey, ...(model ? { model } : {}), ...(baseURL ? { baseURL } : {}) }); + const prompts = await loadPromptTexts(); + + console.log('\n=== 生成一次 → 对同一批做 复核关/开（干净 A/B）==='); + const { baseline, optimized } = await runCleanAB(corpus.cases, prompts, mkClient(), mkClient()); + const results: ConfigResult[] = [baseline, optimized]; const date = new Date().toISOString().slice(0, 10); const md = reportMarkdown(results, corpus.cases, { @@ -264,7 +324,13 @@ async function main(): Promise { console.log(`\n报告已写入 ${out}`); } -main().catch((e: unknown) => { - console.error(e instanceof Error ? e.message : String(e)); - process.exit(1); -}); +// Run only when invoked as the entry (node build/eval/run.js), NOT when imported +// by a test — importing must not kick off a live eval / exit the process. +const invokedDirectly = + process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedDirectly) { + main().catch((e: unknown) => { + console.error(e instanceof Error ? e.message : String(e)); + process.exit(1); + }); +}