[Fix] Mutation gate fails for stale and current pull requests

.github/workflows/mutation-testing.yml

@@ -25,12 +25,11 @@ jobs:
                   echo "## Changed-code mutation testing" >> "$GITHUB_STEP_SUMMARY"
                   echo "Mutation testing was enforced on each pull request before it entered the merge queue." >> "$GITHUB_STEP_SUMMARY"
 
-            - name: Checkout pull request head
+            - name: Checkout pull request merge result
               if: github.event_name == 'pull_request'
               uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
               with:
-                  repository: ${{ github.event.pull_request.head.repo.full_name }}
-                  ref: ${{ github.event.pull_request.head.sha }}
+                  ref: ${{ github.sha }}
                   fetch-depth: 0
                   persist-credentials: false
 
@@ -55,14 +54,26 @@ jobs:
               if: github.event_name == 'pull_request'
               env:
                   BASE_SHA: ${{ github.event.pull_request.base.sha }}
-                  HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+                  HEAD_SHA: ${{ github.sha }}
               run: node scripts/stryker-diff.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
 
             - name: Upload mutation reports
+              id: mutation_report
               if: always() && github.event_name == 'pull_request'
               uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
               with:
                   name: changed-code-mutation-report
                   path: reports/mutation/
                   if-no-files-found: ignore
                   retention-days: 7
+
+            - name: Link mutation report artifact
+              if: always() && github.event_name == 'pull_request' && steps.mutation_report.outputs.artifact-url != ''
+              env:
+                  ARTIFACT_URL: ${{ steps.mutation_report.outputs.artifact-url }}
+              run: |
+                  {
+                      echo ""
+                      echo "### Download mutation reports"
+                      echo "[Open the changed-code-mutation-report artifact]($ARTIFACT_URL), then open the package's mutation.html file."
+                  } >> "$GITHUB_STEP_SUMMARY"

scripts/stryker-diff.mjs

@@ -304,14 +304,23 @@ export function preferDirectTestFiles(testFiles, sourceFiles) {
 	return direct.length > 0 ? direct : testFiles
 }
 
-function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) {
+export function resolveVitestBinary(repoRoot, packageEntry) {
+	const packageRoot = path.join(repoRoot, packageEntry.root)
+	const runRoot = path.join(repoRoot, packageEntry.runRoot ?? packageEntry.root)
+	const candidates = [...new Set([runRoot, packageRoot, repoRoot])].map((root) =>
+		path.join(root, "node_modules/.bin/vitest"),
+	)
+	return candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates.at(-1)
+}
+
+export function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) {
 	const packageRoot = path.join(repoRoot, packageEntry.root)
 	const runRoot = path.join(repoRoot, packageEntry.runRoot ?? packageEntry.root)
 	const outputFile = path.join(reportDirectory, "vitest-related.json")
 	const configFile = path.relative(runRoot, path.join(packageRoot, packageEntry.vitestConfig)).replaceAll("\\", "/")
 	const sourceFiles = [...new Set(packageEntry.selectors.map(selectorFile))]
 	const result = spawnSync(
-		path.join(repoRoot, "node_modules/.bin/vitest"),
+		resolveVitestBinary(repoRoot, packageEntry),
 		["related", ...sourceFiles, "--run", "--config", configFile, "--reporter=json", `--outputFile=${outputFile}`],
 		{
 			cwd: runRoot,
@@ -325,6 +334,9 @@ function discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory) {
 	if (result.error?.code === "ETIMEDOUT") {
 		throw new Error(`${packageEntry.id} related-test discovery exceeded 5 minutes`)
 	}
+	if (result.error) {
+		throw new Error(`${packageEntry.id} related-test discovery could not start: ${result.error.message}`)
+	}
 	if (result.status !== 0) {
 		throw new Error(
 			`${packageEntry.id} related-test discovery failed:\n${stripAnsi(`${result.stdout ?? ""}${result.stderr ?? ""}`).trim()}`,
@@ -378,6 +390,11 @@ function runStryker(repoRoot, packageEntry, reportRoot, dryRunOnly) {
 			`${packageEntry.id} mutation run exceeded 12 minutes. Split the PR or obtain a maintainer-reviewed narrow exclusion.`,
 		)
 	}
+	if (result.error) {
+		throw new Error(
+			`${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} could not start: ${result.error.message}`,
+		)
+	}
 	if (result.status !== 0) {
 		throw new Error(
 			`${packageEntry.id} Stryker ${dryRunOnly ? "preflight" : "run"} failed:\n${stripAnsi(output).trim()}`,
@@ -405,17 +422,19 @@ export function mutantCounts(report) {
 	return counts
 }
 
-function escapeWorkflowValue(value) {
-	return String(value)
-		.replaceAll("%", "%25")
-		.replaceAll("\r", "%0D")
-		.replaceAll("\n", "%0A")
-		.replaceAll(":", "%3A")
-		.replaceAll(",", "%2C")
+function escapeWorkflowData(value) {
+	return String(value).replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A")
+}
+
+function escapeWorkflowProperty(value) {
+	return escapeWorkflowData(value).replaceAll(":", "%3A").replaceAll(",", "%2C")
 }
 
-export function formatAnnotations(blockingMutants, packageRoot) {
-	const perFile = new Map()
+export function formatAnnotationCommand(annotation) {
+	return `::error file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test gap::${escapeWorkflowData(annotation.message)}`
+}
+
+export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) {
 	const annotations = []
 
 	for (const mutant of blockingMutants.sort((left, right) => {
@@ -424,9 +443,8 @@ export function formatAnnotations(blockingMutants, packageRoot) {
 	})) {
 		const repositoryPath = path.posix.join(packageRoot, mutant.filePath.replaceAll("\\", "/"))
 		const key = `${repositoryPath}:${mutant.location.start.line}`
-		const fileCount = perFile.get(repositoryPath) ?? 0
-		if (annotations.some((annotation) => annotation.key === key) || fileCount >= 7 || annotations.length >= 20)
-			continue
+		const fileCount = state.perFile.get(repositoryPath) ?? 0
+		if (annotations.some((annotation) => annotation.key === key) || fileCount >= 7 || state.total >= 20) continue
 
 		const replacement = String(mutant.replacement ?? "")
 			.replace(/\s+/g, " ")
@@ -438,17 +456,59 @@ export function formatAnnotations(blockingMutants, packageRoot) {
 			line: mutant.location.start.line,
 			message:
 				`${mutant.status} ${mutant.mutatorName} mutant${replacement ? ` (replacement: ${replacement})` : ""}. ` +
-				"Add or strengthen a focused test that fails under this mutation, or add a maintainer-approved targeted exclusion with a reason.",
+				"See the job summary for the complete list and resolution guidance.",
 		})
-		perFile.set(repositoryPath, fileCount + 1)
+		state.perFile.set(repositoryPath, fileCount + 1)
+		state.total++
 	}
 
 	return annotations
 }
 
-function appendSummary(rows, failures) {
-	if (!process.env.GITHUB_STEP_SUMMARY) return
+function markdownCell(value) {
+	return String(value ?? "—")
+		.replace(/\s+/g, " ")
+		.trim()
+		.replaceAll("|", "\\|")
+		.slice(0, 120)
+}
+
+export function testsFromMutationReport(report, fallback = []) {
+	const testFiles = Object.keys(report.testFiles ?? {})
+	return testFiles.length > 0 ? testFiles : fallback
+}
+
+export function formatBlockingMutants(blockingMutants, packageRoot) {
+	const grouped = new Map()
+	for (const mutant of [...blockingMutants].sort((left, right) => {
+		const pathOrder = left.filePath.localeCompare(right.filePath)
+		return pathOrder || left.location.start.line - right.location.start.line
+	})) {
+		const repositoryPath = path.posix.join(packageRoot, mutant.filePath.replaceAll("\\", "/"))
+		const group = grouped.get(repositoryPath) ?? []
+		group.push(mutant)
+		grouped.set(repositoryPath, group)
+	}
+
+	const lines = []
+	for (const [filePath, mutants] of grouped) {
+		lines.push(
+			`#### \`${filePath}\``,
+			"",
+			"| Line | Status | Mutator | Replacement |",
+			"| ---: | --- | --- | --- |",
+		)
+		for (const mutant of mutants) {
+			lines.push(
+				`| ${mutant.location.start.line} | ${markdownCell(mutant.status)} | ${markdownCell(mutant.mutatorName)} | ${markdownCell(mutant.replacement)} |`,
+			)
+		}
+		lines.push("")
+	}
+	return lines
+}
 
+export function formatSummary(rows, failures, manifest = {}) {
 	const lines = [
 		"## Changed-code mutation testing",
 		"",
@@ -461,8 +521,89 @@ function appendSummary(rows, failures) {
 		)
 	}
 	if (rows.length === 0) lines.push("| — | 0 | 0 | 0 | 0 | 0 | 0 | Not applicable |")
-	if (failures.length > 0) lines.push("", ...failures.map((failure) => `- ${failure}`))
-	fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`)
+
+	if (rows.length > 0) {
+		lines.push("", "### Focused tests")
+		for (const row of rows) {
+			const cwd = row.runRoot ?? row.root
+			if (row.testFiles?.length > 0) {
+				lines.push(`- **${row.id}** (cwd \`${cwd}\`): ${row.testFiles.map((file) => `\`${file}\``).join(", ")}`)
+			} else {
+				lines.push(
+					`- **${row.id}** (cwd \`${cwd}\`): the mutation run did not complete far enough to report its selected tests; use the exact reproduction command below.`,
+				)
+			}
+		}
+	}
+
+	const blockingRows = rows.filter((row) => row.blocking?.length > 0)
+	if (blockingRows.length > 0) {
+		lines.push(
+			"",
+			"### All surviving and uncovered mutants",
+			"",
+			"Annotations highlight up to 20 unique locations (maximum 7 per file). This summary lists every blocking mutant.",
+			"",
+		)
+		for (const row of blockingRows) {
+			lines.push(`### ${row.id}`, "", ...formatBlockingMutants(row.blocking, row.runRoot ?? row.root))
+		}
+		lines.push(
+			"### Resolve a mutation gap",
+			"",
+			"Add or strengthen a focused test that fails under the mutation. If the mutant is equivalent, request maintainer approval for the narrowest mutator-specific exclusion and explain why it cannot change behavior:",
+			"",
+			"```ts",
+			"// Stryker disable next-line ConditionalExpression: normalized input cannot reach the alternate branch",
+			"const result = condition ? value : fallback",
+			"```",
+			"",
+			"Broad `all` exclusions and exclusions without a concrete reason are rejected by the gate.",
+		)
+	}
+
+	if (manifest.baseSha && manifest.headSha) {
+		lines.push(
+			"",
+			"### Reproduce locally",
+			"",
+			"From a full checkout containing both commits:",
+			"",
+			"```bash",
+			"pnpm install --frozen-lockfile",
+			`node scripts/stryker-diff.mjs ci --base ${manifest.baseSha} --head ${manifest.headSha}`,
+			"```",
+		)
+	}
+
+	if (rows.length > 0) {
+		lines.push("", "### Mutation reports", "")
+		for (const row of rows) lines.push(`- **${row.id}:** \`${row.reportPath}\``)
+		lines.push(
+			"",
+			"The workflow uploads generated reports in the `changed-code-mutation-report` artifact. A direct artifact link appears below after upload.",
+		)
+	}
+
+	if (failures.length > 0) {
+		lines.push(
+			"",
+			"### Failures",
+			"",
+			...failures.map((failure) => {
+				const detail =
+					failure.length > 4_000 ? `${failure.slice(0, 4_000)}\n[truncated; see the step log]` : failure
+				return `- ${detail.replaceAll("\n", "\n  ")}`
+			}),
+		)
+	}
+
+	return `${lines.join("\n")}\n`
+}
+
+function appendSummary(rows, failures, manifest) {
+	if (!process.env.GITHUB_STEP_SUMMARY) return
+	fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, failures, manifest))
 }
 
 export function evaluateReport(report, packageEntry) {
@@ -491,9 +632,13 @@ export function evaluateReport(report, packageEntry) {
 export function runManifest(repoRoot, manifest, reportRoot) {
 	const rows = []
 	const failures = []
+	const annotationState = { total: 0, perFile: new Map() }
 
 	for (const packageEntry of manifest.packages) {
 		let counts
+		const reportPath = path
+			.relative(repoRoot, path.join(reportRoot, packageEntry.id, "mutation.html"))
+			.replaceAll("\\", "/")
 		try {
 			const reportDirectory = path.join(reportRoot, packageEntry.id)
 			fs.mkdirSync(reportDirectory, { recursive: true })
@@ -514,6 +659,11 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 			if (generatedMutants === 0) {
 				rows.push({
 					id: packageEntry.id,
+					root: packageEntry.root,
+					runRoot: packageEntry.runRoot,
+					selectors: packageEntry.selectors,
+					testFiles: packageEntry.testFiles ?? [],
+					reportPath,
 					changedLines: packageEntry.changedExecutableLines,
 					valid: 0,
 					killed: 0,
@@ -526,17 +676,25 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 			}
 
 			runStryker(repoRoot, packageEntry, reportRoot, false)
-			const reportPath = path.join(reportRoot, packageEntry.id, "mutation.json")
-			const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
+			const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json")
+			const report = JSON.parse(fs.readFileSync(jsonReportPath, "utf8"))
+			packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles)
 			counts = mutantCounts(report)
-			for (const annotation of formatAnnotations(counts.blocking, packageEntry.runRoot ?? packageEntry.root)) {
-				console.log(
-					`::error file=${escapeWorkflowValue(annotation.file)},line=${annotation.line},title=Mutation test gap::${escapeWorkflowValue(annotation.message)}`,
-				)
+			for (const annotation of formatAnnotations(
+				counts.blocking,
+				packageEntry.runRoot ?? packageEntry.root,
+				annotationState,
+			)) {
+				console.log(formatAnnotationCommand(annotation))
 			}
 			evaluateReport(report, packageEntry)
 			rows.push({
 				id: packageEntry.id,
+				root: packageEntry.root,
+				runRoot: packageEntry.runRoot,
+				selectors: packageEntry.selectors,
+				testFiles: packageEntry.testFiles ?? [],
+				reportPath,
 				changedLines: packageEntry.changedExecutableLines,
 				...counts,
 				result: "Passed",
@@ -545,18 +703,24 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 			failures.push(error.message)
 			rows.push({
 				id: packageEntry.id,
+				root: packageEntry.root,
+				runRoot: packageEntry.runRoot,
+				selectors: packageEntry.selectors,
+				testFiles: packageEntry.testFiles ?? [],
+				reportPath,
 				changedLines: packageEntry.changedExecutableLines,
 				valid: counts?.valid ?? 0,
 				killed: counts?.killed ?? 0,
 				timeout: counts?.timeout ?? 0,
 				survived: counts?.survived ?? 0,
 				noCoverage: counts?.noCoverage ?? 0,
+				blocking: counts?.blocking ?? [],
 				result: "Failed",
 			})
 		}
 	}
 
-	appendSummary(rows, failures)
+	appendSummary(rows, failures, manifest)
 	if (failures.length > 0) throw new Error(failures.join("\n"))
 	return rows
 }
@@ -579,7 +743,7 @@ function main() {
 	const reportRoot = path.resolve(repoRoot, argument("--reports") ?? "reports/mutation")
 	const manifest = selectFromGit(repoRoot, baseSha, headSha)
 	if (manifest.packages.length === 0) {
-		appendSummary([], [])
+		appendSummary([], [], manifest)
 		console.log("No changed executable lines in mutation-tested packages; mutation testing is not applicable.")
 		return
 	}