diff --git a/src/review/loosening-recs.ts b/src/review/loosening-recs.ts index f8baa3c73e..06972f979a 100644 --- a/src/review/loosening-recs.ts +++ b/src/review/loosening-recs.ts @@ -11,6 +11,7 @@ // advisor's apply path. PURE — no IO; the caller supplies the loop's state. import type { TuningRec } from "./auto-tune"; import type { SatisfactionFloorLooseningProposal } from "../services/satisfaction-floor-loosening"; +import type { KnobLooseningProposal } from "../services/loosening-knobs"; /** The advisor list is per-project elsewhere; the satisfaction floor is deployment-global, so its recs use * this fixed pseudo-project label rather than impersonating any repo. */ @@ -59,3 +60,21 @@ export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecI } return recs; } + +/** + * Recs for REPORT-ONLY registry knobs (#8159): the evidence surfaces exactly like a live knob's proposal, + * but the action line states plainly that this knob's apply is not wired — enabling it is a per-knob, + * reviewed decision (its consumption plumbing changes real authority), never a flag flip. Same hard + * boundary: no overridePayload, ever. + */ +export function buildReportOnlyKnobRecs(proposals: readonly KnobLooseningProposal[]): TuningRec[] { + return proposals.map((proposal) => ({ + project: `global:${proposal.knobId}`, + severity: "good" as const, + message: + `Backtest-cleared LOOSENING evidence for ${proposal.knobId} (report-only): ${proposal.currentValue} → ${proposal.proposedValue}. ` + + `Visible split ${proposal.visible.verdict} (${proposal.visibleCases} case(s), precision ${pct(proposal.visible.baseline.precision)} → ${pct(proposal.visible.candidate.precision)}); ` + + `held-out split ${proposal.heldOut.verdict} (${proposal.heldOutCases} case(s)). ` + + "This knob has no override consumer yet — applying requires shipping its consumption plumbing as its own reviewed change.", + })); +} diff --git a/src/review/selftune-wire.ts b/src/review/selftune-wire.ts index 9ad301b859..e6055a9bb7 100644 --- a/src/review/selftune-wire.ts +++ b/src/review/selftune-wire.ts @@ -41,8 +41,8 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { errorMessage } from "../utils/json"; import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune"; -import { buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; -import { loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; +import { buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; +import { loadReportOnlyKnobProposals, loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply"; /** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy @@ -161,7 +161,12 @@ export async function runSelfTune(env: Env): Promise { // design — runAutoApplyRecommendations below only ever consumes recs carrying a TIGHTENING // overridePayload, so these are report-only here and can never be promoted by the apply path. // Appended once per pass (deployment-global state), on the first repo's iteration. - if (repoFullName === repos[0]) recs.push(...buildSatisfactionFloorLooseningRecs(await loadSatisfactionFloorRecState(env, nowMs))); + if (repoFullName === repos[0]) { + recs.push(...buildSatisfactionFloorLooseningRecs(await loadSatisfactionFloorRecState(env, nowMs))); + // #8159: report-only registry knobs surface their evidence in the same pass -- payload-less, so + // the apply path below ignores them identically. + recs.push(...buildReportOnlyKnobRecs(await loadReportOnlyKnobProposals(env, nowMs))); + } // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow- // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass. await runAutoApplyRecommendations(env as unknown as StorageEnv, { diff --git a/src/services/loosening-knobs.ts b/src/services/loosening-knobs.ts new file mode 100644 index 0000000000..502f9b4eb4 --- /dev/null +++ b/src/services/loosening-knobs.ts @@ -0,0 +1,130 @@ +// Bounded loosenable-knob registry (#8159, sub-issue of epic #8121). The #8121 narrow start hardcoded ONE +// loosenable value (the satisfaction floor); this registry generalizes the shape the same way +// KNOWN_THRESHOLDS (threshold-backtest.ts) and KNOWN_LOGIC_RULES (backtest-logic-check-core.ts) declare +// their surfaces: each knob is a declarative entry — rule id, candidate steps, hard bounds, split +// discipline — evaluated by ONE generic function, never per-knob bespoke loops. +// +// Every knob keeps the narrow start's invariants verbatim: smallest-step-first, strictly `improved` on the +// visible split AND non-`regressed` on the deterministic held-out split, a hard safety minimum no evidence +// can cross, and never-on-noise sample floors. A knob additionally declares whether its apply path is LIVE +// (an override consumer exists) or REPORT-ONLY (proposals surface with full evidence, but nothing may be +// written until the consumption plumbing ships — adding a consumer is a deliberate, per-knob decision, not +// a registry edit side effect). +import { + buildConfidenceThresholdClassifier, + compareBacktestScores, + scoreBacktest, + splitBacktestCorpus, + type BacktestCase, + type BacktestComparison, +} from "@loopover/engine"; +import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "./linked-issue-satisfaction"; +import { DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE } from "../rules/advisory"; + +export type LoosenableKnob = { + /** Stable id — used in override flag keys, audit events, and advisor labels. Never rename. */ + knobId: string; + ruleId: string; + shippedValue: number; + /** Candidate loosened values, nearest-to-shipped first — the smallest evidence-cleared step wins. */ + candidates: readonly number[]; + /** No backtest result, however good, may loosen below this. */ + hardMinimum: number; + minVisibleCases: number; + minHeldOutCases: number; + heldOutFraction: number; + /** Fixed per-knob split seed — held-out membership must never reshuffle between evaluations. */ + splitSeed: string; + /** `live`: an override consumer exists and the apply path may write. `report_only`: proposals surface + * (advisor/status) but the apply path REFUSES — flipping a knob to live requires shipping its + * consumption plumbing first, reviewed on its own. */ + applyMode: "live" | "report_only"; +}; + +export const LOOSENABLE_KNOBS: Readonly> = Object.freeze({ + // #8121's approved narrow start — fully live (override consumed by runLoopOverLinkedIssueSatisfaction). + // Values and seed are IDENTICAL to the pre-registry constants: behavior and held-out membership are + // byte-stable across this refactor. + satisfaction_floor: { + knobId: "satisfaction_floor", + ruleId: "linked_issue_scope_mismatch", + shippedValue: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR, + candidates: [0.45, 0.4, 0.35, 0.3], + hardMinimum: 0.3, + minVisibleCases: 20, + minHeldOutCases: 5, + heldOutFraction: 0.25, + splitSeed: "satisfaction-floor-loosening-v1", + applyMode: "live", + }, + // The AI close-confidence floor (#8159's second knob). Its corpus (ai_consensus_defect — including the + // #8157 backfilled decision-level history) is real TODAY, so proposals carry evidence now — but + // loosening it means MORE auto-closes, a direct gate-authority change, so it enters REPORT-ONLY: no + // override consumer exists yet, and the apply path refuses until that plumbing ships as its own + // reviewed change. Tight bounds by design: two small steps, hard floor 0.85. + ai_review_close_confidence: { + knobId: "ai_review_close_confidence", + ruleId: "ai_consensus_defect", + shippedValue: DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE, + candidates: [0.9, 0.85], + hardMinimum: 0.85, + minVisibleCases: 50, + minHeldOutCases: 12, + heldOutFraction: 0.25, + splitSeed: "ai-close-confidence-loosening-v1", + applyMode: "report_only", + }, +}); + +export type KnobLooseningProposal = { + knobId: string; + ruleId: string; + currentValue: number; + proposedValue: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; +}; + +/** + * Evaluate whether `knob` can be safely loosened from `currentValue` — the generic form of the #8121 + * narrow start's gate, parameterized by the registry entry and nothing else: the smallest candidate step + * below `currentValue` (never below the knob's hard minimum) whose backtest verdict is strictly + * `"improved"` on the visible split AND non-`"regressed"` on the held-out split. Null when the corpus is + * too small, no candidate qualifies, or the current value already sits at/below the hard minimum. Pure and + * deterministic — same knob + corpus + value ⇒ same proposal. + */ +export function evaluateKnobLoosening( + knob: LoosenableKnob, + cases: readonly BacktestCase[], + currentValue: number = knob.shippedValue, +): KnobLooseningProposal | null { + const { visible, heldOut } = splitBacktestCorpus(cases, knob.heldOutFraction, knob.splitSeed); + if (visible.length < knob.minVisibleCases || heldOut.length < knob.minHeldOutCases) return null; + + for (const candidate of knob.candidates) { + if (candidate >= currentValue || candidate < knob.hardMinimum) continue; + const visibleComparison = compareOnSlice(knob.ruleId, visible, currentValue, candidate); + if (visibleComparison.verdict !== "improved") continue; + const heldOutComparison = compareOnSlice(knob.ruleId, heldOut, currentValue, candidate); + if (heldOutComparison.verdict === "regressed") continue; + return { + knobId: knob.knobId, + ruleId: knob.ruleId, + currentValue, + proposedValue: candidate, + visibleCases: visible.length, + heldOutCases: heldOut.length, + visible: visibleComparison, + heldOut: heldOutComparison, + }; + } + return null; +} + +function compareOnSlice(ruleId: string, slice: readonly BacktestCase[], currentValue: number, candidate: number): BacktestComparison { + const baseline = scoreBacktest(ruleId, slice, buildConfidenceThresholdClassifier(currentValue)); + const proposed = scoreBacktest(ruleId, slice, buildConfidenceThresholdClassifier(candidate)); + return compareBacktestScores(baseline, proposed); +} diff --git a/src/services/satisfaction-floor-loosening-run.ts b/src/services/satisfaction-floor-loosening-run.ts index 5c6641fea8..1a443273a6 100644 --- a/src/services/satisfaction-floor-loosening-run.ts +++ b/src/services/satisfaction-floor-loosening-run.ts @@ -14,6 +14,7 @@ import { buildBacktestCorpus } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; +import { evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobLooseningProposal } from "./loosening-knobs"; import { evaluateSatisfactionFloorLoosening, SATISFACTION_FLOOR_HARD_MINIMUM, @@ -273,3 +274,24 @@ export async function loadSatisfactionFloorStatus(env: Env): Promise { + const proposals: KnobLooseningProposal[] = []; + for (const knob of Object.values(LOOSENABLE_KNOBS)) { + if (knob.applyMode !== "report_only") continue; + try { + const { fired, overrides } = await createSignalStore(env).queryRuleHistory(knob.ruleId, nowMs - CORPUS_LOOKBACK_MS); + const proposal = evaluateKnobLoosening(knob, buildBacktestCorpus(knob.ruleId, fired, overrides)); + if (proposal) proposals.push(proposal); + } catch { + /* one knob's read blip must not hide the others' proposals */ + } + } + return proposals; +} diff --git a/src/services/satisfaction-floor-loosening.ts b/src/services/satisfaction-floor-loosening.ts index 1715a92040..c5c6ac15f6 100644 --- a/src/services/satisfaction-floor-loosening.ts +++ b/src/services/satisfaction-floor-loosening.ts @@ -11,15 +11,9 @@ // via the SignalStore). Scoped to exactly this one scalar; generalizing to other loosenable knobs is the // rest of epic #8121, decomposed separately, and requires its own explicit approval per that epic's // Boundaries. -import { - buildConfidenceThresholdClassifier, - compareBacktestScores, - scoreBacktest, - splitBacktestCorpus, - type BacktestCase, - type BacktestComparison, -} from "@loopover/engine"; +import type { BacktestCase, BacktestComparison } from "@loopover/engine"; import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "./linked-issue-satisfaction"; +import { evaluateKnobLoosening, LOOSENABLE_KNOBS } from "./loosening-knobs"; export const SATISFACTION_FLOOR_RULE_ID = "linked_issue_scope_mismatch"; @@ -52,42 +46,22 @@ export type SatisfactionFloorLooseningProposal = { heldOut: BacktestComparison; }; -/** - * Evaluate whether the satisfaction confidence floor can be safely loosened, per #8121's approved gate: - * the smallest candidate step below `currentFloor` (never below {@link SATISFACTION_FLOOR_HARD_MINIMUM}) - * whose backtest verdict is strictly `"improved"` on the visible split AND non-`"regressed"` on the - * held-out split. Returns null when the corpus is too small, no candidate qualifies, or `currentFloor` is - * already at/below the hard minimum (a previously-applied loosening never compounds past it). Pure and - * deterministic — same corpus + floor ⇒ same proposal. - */ export function evaluateSatisfactionFloorLoosening( cases: readonly BacktestCase[], currentFloor: number = LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR, ): SatisfactionFloorLooseningProposal | null { - const { visible, heldOut } = splitBacktestCorpus(cases, SATISFACTION_FLOOR_HELD_OUT_FRACTION, SATISFACTION_FLOOR_SPLIT_SEED); - if (visible.length < SATISFACTION_FLOOR_MIN_VISIBLE_CASES || heldOut.length < SATISFACTION_FLOOR_MIN_HELD_OUT_CASES) return null; - - for (const candidate of SATISFACTION_FLOOR_LOOSENING_CANDIDATES) { - if (candidate >= currentFloor || candidate < SATISFACTION_FLOOR_HARD_MINIMUM) continue; - const visibleComparison = compareOnSlice(visible, currentFloor, candidate); - if (visibleComparison.verdict !== "improved") continue; - const heldOutComparison = compareOnSlice(heldOut, currentFloor, candidate); - if (heldOutComparison.verdict === "regressed") continue; - return { - ruleId: SATISFACTION_FLOOR_RULE_ID, - currentFloor, - proposedFloor: candidate, - visibleCases: visible.length, - heldOutCases: heldOut.length, - visible: visibleComparison, - heldOut: heldOutComparison, - }; - } - return null; -} - -function compareOnSlice(slice: readonly BacktestCase[], currentFloor: number, candidate: number): BacktestComparison { - const baseline = scoreBacktest(SATISFACTION_FLOOR_RULE_ID, slice, buildConfidenceThresholdClassifier(currentFloor)); - const proposed = scoreBacktest(SATISFACTION_FLOOR_RULE_ID, slice, buildConfidenceThresholdClassifier(candidate)); - return compareBacktestScores(baseline, proposed); + // #8159: delegates to the generic knob evaluator with the registry entry whose values/seed are pinned + // (by test) to this module's own legacy constants -- behavior and held-out membership are byte-stable + // across the refactor. This wrapper only re-shapes the field names the #8121 consumers already use. + const proposal = evaluateKnobLoosening(LOOSENABLE_KNOBS.satisfaction_floor!, cases, currentFloor); + if (!proposal) return null; + return { + ruleId: SATISFACTION_FLOOR_RULE_ID, + currentFloor: proposal.currentValue, + proposedFloor: proposal.proposedValue, + visibleCases: proposal.visibleCases, + heldOutCases: proposal.heldOutCases, + visible: proposal.visible, + heldOut: proposal.heldOut, + }; } diff --git a/test/unit/loosening-knobs.test.ts b/test/unit/loosening-knobs.test.ts new file mode 100644 index 0000000000..695b064f6e --- /dev/null +++ b/test/unit/loosening-knobs.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { splitBacktestCorpus, type BacktestCase } from "@loopover/engine"; +import { evaluateKnobLoosening, LOOSENABLE_KNOBS, type LoosenableKnob } from "../../src/services/loosening-knobs"; +import { + SATISFACTION_FLOOR_HARD_MINIMUM, + SATISFACTION_FLOOR_HELD_OUT_FRACTION, + SATISFACTION_FLOOR_LOOSENING_CANDIDATES, + SATISFACTION_FLOOR_MIN_HELD_OUT_CASES, + SATISFACTION_FLOOR_MIN_VISIBLE_CASES, + SATISFACTION_FLOOR_RULE_ID, + SATISFACTION_FLOOR_SPLIT_SEED, +} from "../../src/services/satisfaction-floor-loosening"; +import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction"; +import { DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE } from "../../src/rules/advisory"; +import { buildReportOnlyKnobRecs } from "../../src/review/loosening-recs"; + +const AI_KNOB = LOOSENABLE_KNOBS.ai_review_close_confidence!; + +describe("LOOSENABLE_KNOBS registry invariants (#8159)", () => { + it("pins the satisfaction knob to the #8121 narrow start's exact values and seed — behavior and held-out membership stay byte-stable", () => { + expect(LOOSENABLE_KNOBS.satisfaction_floor).toEqual({ + knobId: "satisfaction_floor", + ruleId: SATISFACTION_FLOOR_RULE_ID, + shippedValue: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR, + candidates: SATISFACTION_FLOOR_LOOSENING_CANDIDATES, + hardMinimum: SATISFACTION_FLOOR_HARD_MINIMUM, + minVisibleCases: SATISFACTION_FLOOR_MIN_VISIBLE_CASES, + minHeldOutCases: SATISFACTION_FLOOR_MIN_HELD_OUT_CASES, + heldOutFraction: SATISFACTION_FLOOR_HELD_OUT_FRACTION, + splitSeed: SATISFACTION_FLOOR_SPLIT_SEED, + applyMode: "live", + }); + }); + + it("pins the close-confidence knob to the shipped default, tight bounds, and REPORT-ONLY apply mode", () => { + expect(AI_KNOB.shippedValue).toBe(DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE); + expect(AI_KNOB.ruleId).toBe("ai_consensus_defect"); + expect(AI_KNOB.applyMode).toBe("report_only"); + expect(AI_KNOB.hardMinimum).toBe(0.85); + }); + + it("every entry satisfies the structural safety invariants: candidates strictly below shipped, at/above the hard minimum, descending; ids and seeds unique", () => { + const knobs = Object.values(LOOSENABLE_KNOBS); + for (const knob of knobs) { + expect(knob.candidates.length).toBeGreaterThan(0); + for (const candidate of knob.candidates) { + expect(candidate).toBeLessThan(knob.shippedValue); + expect(candidate).toBeGreaterThanOrEqual(knob.hardMinimum); + } + expect([...knob.candidates].sort((a, b) => b - a)).toEqual([...knob.candidates]); // nearest-first + expect(knob.minVisibleCases).toBeGreaterThan(0); + expect(knob.minHeldOutCases).toBeGreaterThan(0); + expect(["live", "report_only"]).toContain(knob.applyMode); + } + expect(new Set(knobs.map((knob) => knob.knobId)).size).toBe(knobs.length); + expect(new Set(knobs.map((knob) => knob.splitSeed)).size).toBe(knobs.length); + for (const [key, knob] of Object.entries(LOOSENABLE_KNOBS)) expect(key).toBe(knob.knobId); + }); +}); + +// Fixture strategy mirrors the satisfaction suite: probe the real splitter for slice membership under THIS +// knob's seed/rule, then assign confidence/label per slice. +function aiCase(targetKey: string, confidence: number, label: "reversed" | "confirmed"): BacktestCase { + return { + ruleId: AI_KNOB.ruleId, + targetKey, + outcome: "close", + label, + firedAt: "2026-06-01T00:00:00.000Z", + decidedAt: "2026-06-02T00:00:00.000Z", + metadata: { confidence }, + }; +} + +const POOL = Array.from({ length: 400 }, (_, i) => `acme/widgets#${i + 1}`); +const probe = POOL.map((key) => aiCase(key, 0.99, "confirmed")); +const { visible, heldOut } = splitBacktestCorpus(probe, AI_KNOB.heldOutFraction, AI_KNOB.splitSeed); +const visibleKeys = visible.map((c) => c.targetKey); +const heldOutKeys = heldOut.map((c) => c.targetKey); + +function aiLooseningFriendlyCorpus(): BacktestCase[] { + const cases: BacktestCase[] = []; + // Borderline firings a human CONFIRMED at confidence 0.91 (between candidate 0.9 and shipped 0.93): + // baseline predicts them reversed (false positives); candidate 0.9 stops firing them — precision improves. + for (const key of visibleKeys.slice(0, AI_KNOB.minVisibleCases + 6)) cases.push(aiCase(key, 0.91, "confirmed")); + for (const key of heldOutKeys.slice(0, AI_KNOB.minHeldOutCases + 3)) cases.push(aiCase(key, 0.91, "confirmed")); + // A deep-low reversed anchor per slice keeps a true positive on both sides of every comparison. + cases.push(aiCase(visibleKeys[AI_KNOB.minVisibleCases + 10]!, 0.5, "reversed")); + cases.push(aiCase(heldOutKeys[AI_KNOB.minHeldOutCases + 6]!, 0.5, "reversed")); + return cases; +} + +describe("evaluateKnobLoosening on the close-confidence knob (#8159)", () => { + it("proposes the smallest candidate step with full evidence when both splits support it", () => { + const proposal = evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus()); + expect(proposal).not.toBeNull(); + expect(proposal!.knobId).toBe("ai_review_close_confidence"); + expect(proposal!.currentValue).toBe(DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE); + expect(proposal!.proposedValue).toBe(0.9); + expect(proposal!.visible.verdict).toBe("improved"); + expect(proposal!.heldOut.verdict).not.toBe("regressed"); + }); + + it("never loosens on a sample below THIS knob's own (higher) floors", () => { + const thin = [ + ...visibleKeys.slice(0, AI_KNOB.minVisibleCases - 1).map((key) => aiCase(key, 0.91, "confirmed")), + ...heldOutKeys.slice(0, AI_KNOB.minHeldOutCases + 3).map((key) => aiCase(key, 0.91, "confirmed")), + ]; + expect(evaluateKnobLoosening(AI_KNOB, thin)).toBeNull(); + }); + + it("refuses to step below the hard minimum even from an already-loosened current value", () => { + expect(evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus(), AI_KNOB.hardMinimum)).toBeNull(); + }); +}); + +describe("buildReportOnlyKnobRecs (#8159)", () => { + it("surfaces the evidence with the report-only action line and NEVER a payload", () => { + const proposal = evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus())!; + const recs = buildReportOnlyKnobRecs([proposal]); + expect(recs).toHaveLength(1); + expect(recs[0]!.project).toBe("global:ai_review_close_confidence"); + expect(recs[0]!.severity).toBe("good"); + expect(recs[0]!.message).toContain(`${DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE} → 0.9`); + expect(recs[0]!.message).toContain("no override consumer yet"); + expect(recs[0]!.overridePayload).toBeUndefined(); + expect(buildReportOnlyKnobRecs([])).toEqual([]); + }); +}); diff --git a/test/unit/satisfaction-floor-loosening-run.test.ts b/test/unit/satisfaction-floor-loosening-run.test.ts index 29b21ca1a2..44609fe4f2 100644 --- a/test/unit/satisfaction-floor-loosening-run.test.ts +++ b/test/unit/satisfaction-floor-loosening-run.test.ts @@ -3,6 +3,7 @@ import { splitBacktestCorpus } from "@loopover/engine"; import { getSatisfactionFloorOverride, isSatisfactionFloorAutotuneEnabled, + loadReportOnlyKnobProposals, loadSatisfactionFloorRecState, runSatisfactionFloorLoosening, runScheduledSatisfactionFloorLoosening, @@ -267,3 +268,45 @@ describe("loadSatisfactionFloorRecState (#8160)", () => { expect(state.proposal?.currentFloor).toBe(0.5); }); }); + +describe("loadReportOnlyKnobProposals (#8159)", () => { + it("returns [] on an empty corpus and never touches live-mode knobs", async () => { + expect(await loadReportOnlyKnobProposals(enabledEnv())).toEqual([]); + }); + + it("surfaces a proposal for the close-confidence knob from its own (backfill-shaped) corpus", async () => { + const env = enabledEnv(); + const knob = (await import("../../src/services/loosening-knobs")).LOOSENABLE_KNOBS.ai_review_close_confidence!; + const { splitBacktestCorpus } = await import("@loopover/engine"); + const pool = Array.from({ length: 400 }, (_, i) => ({ + ruleId: knob.ruleId, + targetKey: `acme/widgets#${i + 1}`, + outcome: "close", + label: "confirmed" as const, + firedAt: "2026-06-01T00:00:00.000Z", + decidedAt: "2026-06-02T00:00:00.000Z", + })); + const { visible, heldOut } = splitBacktestCorpus(pool, knob.heldOutFraction, knob.splitSeed); + const store = createSignalStore(env); + const now = Date.now(); + const seed = async (targetKey: string, confidence: number, verdict: "confirmed" | "reversed") => { + await store.recordRuleFired({ ruleId: knob.ruleId, targetKey, outcome: "close", occurredAt: new Date(now - 10_000).toISOString(), metadata: { confidence } }); + await store.recordHumanOverride({ ruleId: knob.ruleId, targetKey, verdict, occurredAt: new Date(now - 5000).toISOString() }); + }; + for (const c of visible.slice(0, knob.minVisibleCases + 6)) await seed(c.targetKey, 0.91, "confirmed"); + for (const c of heldOut.slice(0, knob.minHeldOutCases + 3)) await seed(c.targetKey, 0.91, "confirmed"); + await seed(visible[knob.minVisibleCases + 10]!.targetKey, 0.5, "reversed"); + await seed(heldOut[knob.minHeldOutCases + 6]!.targetKey, 0.5, "reversed"); + + const proposals = await loadReportOnlyKnobProposals(env); + expect(proposals).toHaveLength(1); + expect(proposals[0]!.knobId).toBe("ai_review_close_confidence"); + expect(proposals[0]!.proposedValue).toBe(0.9); + }); + + it("a knob's read blip is skipped, not thrown", async () => { + const env = enabledEnv(); + env.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await loadReportOnlyKnobProposals(env)).toEqual([]); + }); +}); diff --git a/test/unit/selftune-wiring.test.ts b/test/unit/selftune-wiring.test.ts index cdd5dc5aea..0c53820cdb 100644 --- a/test/unit/selftune-wiring.test.ts +++ b/test/unit/selftune-wiring.test.ts @@ -237,6 +237,7 @@ describe("runSelfTune — shadow-soak over loopover's own outcome data", () => { it("appends the loosening-loop recs exactly once per pass, on the first repo only (#8160)", async () => { const state = await import("../../src/services/satisfaction-floor-loosening-run"); const spy = vi.spyOn(state, "loadSatisfactionFloorRecState"); + const reportOnlySpy = vi.spyOn(state, "loadReportOnlyKnobProposals"); const env = createTestEnv({ LOOPOVER_REVIEW_SELFTUNE: "true" }); await seedRegisteredRepo(env, "owner/repo", ACTING_AUTONOMY); await seedRegisteredRepo(env, "owner/other", ACTING_AUTONOMY); @@ -246,8 +247,9 @@ describe("runSelfTune — shadow-soak over loopover's own outcome data", () => { await processJob(env, { type: "selftune", requestedBy: "schedule" }); // Two repos in the pass, ONE deployment-global loosening-state read: the recs are appended on the - // first repo's iteration only, never once per repo. + // first repo's iteration only, never once per repo. Same contract for the report-only knob pass (#8159). expect(spy).toHaveBeenCalledTimes(1); + expect(reportOnlySpy).toHaveBeenCalledTimes(1); }); it("FLAG-ON via the processor: a stale in-flight selftune job runs the tick (defense-in-depth gate)", async () => {