diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 8583da841d..18af250abd 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -492,6 +492,7 @@ import { buildSlopAssessment, type SlopBand } from "../signals/slop"; import { copycatWouldActOnPersistedScore } from "../signals/copycat"; import { buildStructuralImprovementAssessment } from "../signals/improvement"; import { runLoopOverLinkedIssueSatisfaction } from "../services/linked-issue-satisfaction-run"; +import { MAX_BODY_CHARS, MAX_DIFF_CHARS, MAX_ISSUE_TEXT_CHARS } from "../services/linked-issue-satisfaction"; import { decidePublicSurface } from "../signals/settings-preview"; import { buildFocusManifestGuidance, @@ -7553,7 +7554,18 @@ export async function runLinkedIssueSatisfactionForAdvisory( targetKey: `${args.repoFullName}#${args.pr.number}`, outcome: result.result.status, occurredAt: nowIso(), - metadata: { confidence: result.result.confidence }, + // #8129: also capture the bounded raw inputs the assessment was based on — a future backtest + // classify() re-runs new prompt/logic against these and compares to the recorded label; confidence + // alone can only validate threshold changes. Bounds mirror LinkedIssueSatisfactionInput's own + // (reused constants + the exact trim/slice shapes buildLinkedIssueSatisfactionPrompt applies), so + // what's stored is byte-what the assessment actually saw. + metadata: { + confidence: result.result.confidence, + issueText: issueText.trim().slice(0, MAX_ISSUE_TEXT_CHARS), + prTitle: args.pr.title, + prBody: (args.pr.body ?? "").trim().slice(0, MAX_BODY_CHARS), + diff: diff.slice(0, MAX_DIFF_CHARS), + }, }) .catch(() => undefined); } diff --git a/src/services/linked-issue-satisfaction.ts b/src/services/linked-issue-satisfaction.ts index 45d31f839c..5288cab0c7 100644 --- a/src/services/linked-issue-satisfaction.ts +++ b/src/services/linked-issue-satisfaction.ts @@ -29,9 +29,11 @@ export type LinkedIssueSatisfactionStatus = (typeof LINKED_ISSUE_SATISFACTION_ST export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5; const MAX_RATIONALE_LENGTH = 400; -const MAX_ISSUE_TEXT_CHARS = 6000; -const MAX_DIFF_CHARS = 60000; -const MAX_BODY_CHARS = 2000; +// Exported (#8129) so the calibration fired-event capture reuses the assessment's OWN bounds for the raw +// context it stores, instead of drifting behind a second set of hand-maintained limits. +export const MAX_ISSUE_TEXT_CHARS = 6000; +export const MAX_DIFF_CHARS = 60000; +export const MAX_BODY_CHARS = 2000; export type LinkedIssueSatisfactionInput = { /** The already-fetched linked-issue title + body text (grounding already resolved this — see diff --git a/test/unit/linked-issue-satisfaction-run.test.ts b/test/unit/linked-issue-satisfaction-run.test.ts index f02b0239d9..171dd92485 100644 --- a/test/unit/linked-issue-satisfaction-run.test.ts +++ b/test/unit/linked-issue-satisfaction-run.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runLoopOverLinkedIssueSatisfaction, type LinkedIssueSatisfactionRunInput } from "../../src/services/linked-issue-satisfaction-run"; +import { MAX_BODY_CHARS, MAX_DIFF_CHARS } from "../../src/services/linked-issue-satisfaction"; import { BEST_REVIEW_MODELS, RELIABLE_FALLBACK_MODELS } from "../../src/services/ai-review"; import { buildAiReviewDiff, processJob, runLinkedIssueSatisfactionForAdvisory } from "../../src/queue/processors"; import { evaluateGateCheck } from "../../src/rules/advisory"; @@ -481,6 +482,38 @@ describe("runLinkedIssueSatisfactionForAdvisory (processor wiring, #1961/#3906)" expect(history.overrides).toEqual([]); // firing alone is never an override }); + it("captures the bounded raw context (issueText/prTitle/prBody/diff) on the fired signal, truncating an over-limit body (#8129)", async () => { + stubIssueFetch(); + const run = vi.fn(async () => ({ response: satisfactionJson({ status: "unaddressed", confidence: 0.9 }) })); + const env = enabledEnv(run); + const longBody = "x".repeat(MAX_BODY_CHARS + 500); + const longBodyPr = { ...pr, body: longBody }; + await runLinkedIssueSatisfactionForAdvisory(env, { mode: "live", settings: blockMode, advisory: advisory(), repoFullName: "acme/widgets", pr: longBodyPr, author: "alice", files, confirmedContributor: true, installationId: 1 }); + + const history = await createSignalStore(env).queryRuleHistory("linked_issue_scope_mismatch", 0); + expect(history.fired).toHaveLength(1); + const metadata = history.fired[0]?.metadata as Record; + // The stored issue text is the same title+body composite the assessment itself consumed. + expect(metadata.issueText).toContain("Enrich SN74 Gittensor — add SSE stream"); + expect(metadata.issueText).toContain("We need a live SSE stream surface for SN74 Gittensor."); + expect(metadata.prTitle).toBe("Add SSE stream endpoint"); + // The over-limit body is truncated to the assessment's OWN bound, never stored raw. + expect(metadata.prBody).toBe(longBody.slice(0, MAX_BODY_CHARS)); + expect(metadata.prBody).toHaveLength(MAX_BODY_CHARS); + expect(metadata.diff).toBe(buildAiReviewDiff(files).slice(0, MAX_DIFF_CHARS)); + }); + + it("stores an empty prBody (not the string 'null'/'undefined') for a body-less PR (#8129)", async () => { + stubIssueFetch(); + const run = vi.fn(async () => ({ response: satisfactionJson({ status: "unaddressed", confidence: 0.9 }) })); + const env = enabledEnv(run); + const bodylessPr = { ...pr, body: null as unknown as string }; + await runLinkedIssueSatisfactionForAdvisory(env, { mode: "live", settings: blockMode, advisory: advisory(), repoFullName: "acme/widgets", pr: bodylessPr, author: "alice", files, confirmedContributor: true, installationId: 1 }); + + const history = await createSignalStore(env).queryRuleHistory("linked_issue_scope_mismatch", 0); + expect((history.fired[0]?.metadata as Record).prBody).toBe(""); + }); + it("ADVISORY mode records NO fired signal for the same 'unaddressed' verdict (#8101 — no finding, no signal)", async () => { stubIssueFetch(); const run = vi.fn(async () => ({ response: satisfactionJson({ status: "unaddressed", confidence: 0.9 }) }));