|
| 1 | +import { describe, expect, it } from "vitest"; |
| 2 | +import { scoreJudgmentAgreement, UNCORROBORATED_AGREEMENT } from "../../src/review/judgment-agreement"; |
| 3 | + |
| 4 | +// #8834: the per-decision confidence signal. The contract worth pinning is that the score can never read as |
| 5 | +// MORE certain than its weakest input, and that a run with nothing to corroborate it is never scored as |
| 6 | +// unanimous — the same "silence is not certainty" failure #8845 fixed for absent verbalized confidence. |
| 7 | + |
| 8 | +describe("scoreJudgmentAgreement (#8834)", () => { |
| 9 | + const fail = { votedFail: true }; |
| 10 | + const pass = { votedFail: false }; |
| 11 | + |
| 12 | + it("unanimous reviewers score full agreement and keep the verbalized confidence intact", () => { |
| 13 | + expect(scoreJudgmentAgreement([fail, fail], 0.9)).toEqual({ agreement: 1, confidence: 0.9, sampleCount: 2, uncorroborated: false }); |
| 14 | + // Unanimity on the NON-fail stance is equally unanimous — agreement is about reproducibility, not verdict. |
| 15 | + expect(scoreJudgmentAgreement([pass, pass], 0.8)).toMatchObject({ agreement: 1, confidence: 0.8 }); |
| 16 | + }); |
| 17 | + |
| 18 | + it("a split scores below unanimity and drags the combined confidence down with it", () => { |
| 19 | + const split = scoreJudgmentAgreement([fail, pass], 0.9); |
| 20 | + expect(split).toMatchObject({ agreement: 0.5, sampleCount: 2, uncorroborated: false }); |
| 21 | + // 0.5 agreement x 0.9 stated = 0.45: a disagreed-upon judgment is recorded as materially less certain |
| 22 | + // than the same judgment reached unanimously (0.9 above), which is the whole point of the signal. |
| 23 | + expect(split.confidence).toBeCloseTo(0.45, 10); |
| 24 | + expect(split.confidence).toBeLessThan(scoreJudgmentAgreement([fail, fail], 0.9).confidence); |
| 25 | + }); |
| 26 | + |
| 27 | + it("2-of-3 is the modal stance regardless of which side holds it", () => { |
| 28 | + expect(scoreJudgmentAgreement([fail, fail, pass], 1).agreement).toBeCloseTo(2 / 3, 10); |
| 29 | + expect(scoreJudgmentAgreement([pass, pass, fail], 1).agreement).toBeCloseTo(2 / 3, 10); |
| 30 | + expect(scoreJudgmentAgreement([fail, fail, fail], 1).agreement).toBe(1); |
| 31 | + }); |
| 32 | + |
| 33 | + it("a lone run is UNCORROBORATED, never fabricated unanimity — the budget-degraded arm", () => { |
| 34 | + // A single reviewer (single-reviewer plan, or a dual plan whose second leg failed / was budget-cut) |
| 35 | + // cannot corroborate itself. It must record a LOWER confidence than a genuinely agreed judgment, not a |
| 36 | + // flattering 1.0. |
| 37 | + const lone = scoreJudgmentAgreement([fail], 0.9); |
| 38 | + expect(lone).toMatchObject({ agreement: UNCORROBORATED_AGREEMENT, sampleCount: 1, uncorroborated: true }); |
| 39 | + expect(lone.confidence).toBeCloseTo(0.45, 10); |
| 40 | + expect(lone.confidence).toBeLessThan(scoreJudgmentAgreement([fail, fail], 0.9).confidence); |
| 41 | + }); |
| 42 | + |
| 43 | + it("zero samples still refuses to invent a score", () => { |
| 44 | + expect(scoreJudgmentAgreement([], 0.9)).toMatchObject({ agreement: UNCORROBORATED_AGREEMENT, sampleCount: 0, uncorroborated: true }); |
| 45 | + }); |
| 46 | + |
| 47 | + it("clamps and totalizes a hostile verbalized confidence instead of propagating it", () => { |
| 48 | + expect(scoreJudgmentAgreement([fail, fail], 5).confidence).toBe(1); |
| 49 | + expect(scoreJudgmentAgreement([fail, fail], -3).confidence).toBe(0); |
| 50 | + expect(scoreJudgmentAgreement([fail, fail], Number.NaN).confidence).toBe(0); |
| 51 | + expect(scoreJudgmentAgreement([fail, fail], Number.POSITIVE_INFINITY).confidence).toBe(0); |
| 52 | + }); |
| 53 | + |
| 54 | + it("INVARIANT: the combined score never exceeds either input, at any sample shape", () => { |
| 55 | + const shapes = [[fail], [fail, fail], [fail, pass], [pass, pass], [fail, fail, pass], [pass, fail, fail], [fail, fail, fail], []]; |
| 56 | + for (const stated of [0, 0.13, 0.5, 0.93, 1]) { |
| 57 | + for (const shape of shapes) { |
| 58 | + const scored = scoreJudgmentAgreement(shape, stated); |
| 59 | + expect(scored.confidence).toBeLessThanOrEqual(scored.agreement + 1e-12); |
| 60 | + expect(scored.confidence).toBeLessThanOrEqual(stated + 1e-12); |
| 61 | + expect(scored.agreement).toBeGreaterThanOrEqual(0); |
| 62 | + expect(scored.agreement).toBeLessThanOrEqual(1); |
| 63 | + } |
| 64 | + } |
| 65 | + }); |
| 66 | +}); |
0 commit comments