Skip to content

Commit bd6d690

Browse files
committed
feat(enrichment): flag CI check-run retry and long-running signals
1 parent 91f137d commit bd6d690

9 files changed

Lines changed: 438 additions & 2 deletions

File tree

.env.example

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,18 +65,19 @@ GITTENSORY_REVIEW_ENRICHMENT=false
6565
# Current analyzer names:
6666
# dependency,lockfileDrift,secret,license,installScript,heavyDependency,actionPin,eol,redos
6767
# provenance,codeowners,secretLog,assetWeight,typosquat,commitSignature,iacMisconfig,nativeBuild
68-
# history,docCommentDrift,duplication,churnHotspot,blameLink,approvalIntegrity
68+
# history,docCommentDrift,duplication,churnHotspot,blameLink,approvalIntegrity,ciCheckSignals
6969
#
7070
# Profile defaults:
7171
# fast: dependency,lockfileDrift,secret,license,installScript,heavyDependency,actionPin,eol
7272
# redos,provenance,secretLog,typosquat,iacMisconfig,nativeBuild
7373
# balanced (default): dependency,lockfileDrift,secret,license,installScript,heavyDependency
7474
# actionPin,eol,redos,provenance,codeowners,secretLog,assetWeight,typosquat,commitSignature
7575
# iacMisconfig,nativeBuild,history,docCommentDrift,duplication,churnHotspot,blameLink
76-
# approvalIntegrity
76+
# approvalIntegrity,ciCheckSignals
7777
# deep: dependency,lockfileDrift,secret,license,installScript,heavyDependency,actionPin,eol
7878
# redos,provenance,codeowners,secretLog,assetWeight,typosquat,commitSignature,iacMisconfig
7979
# nativeBuild,history,docCommentDrift,duplication,churnHotspot,blameLink,approvalIntegrity
80+
# ciCheckSignals
8081
# END GENERATED REES ANALYZERS
8182

8283
# Submitter-reputation spend control (internal-only): downgrades new/burst/low-rep

apps/gittensory-ui/src/lib/rees-analyzers.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,30 @@ export const REES_ANALYZERS = [
608608
"Structured-fields-only: reads state/commit_id/user.login/submitted_at, never diff or review-body text. Fail-safe on missing token/head SHA/fetch error.",
609609
},
610610
},
611+
{
612+
name: "ciCheckSignals",
613+
title: "CI check-run signals",
614+
category: "history",
615+
cost: "github-light",
616+
defaultEnabled: true,
617+
profiles: ["balanced", "deep"],
618+
requires: ["github-token", "head-sha"],
619+
limits: {
620+
maxCheckRuns: 100,
621+
longRunThresholdMinutes: 15,
622+
},
623+
docs: {
624+
summary:
625+
"Flags a named check that only went green after one or more earlier non-success attempts at the current head commit, and any completed check run whose duration crossed a fixed threshold.",
626+
looksAt:
627+
"The head commit's check-runs (one bounded page), grouped by name and ordered by start time.",
628+
reports:
629+
"Check name and either the count of failed attempts before success, or the run's duration in minutes — never logs or output.",
630+
network: "Calls the GitHub check-runs API once, bounded to one page.",
631+
notes:
632+
"Structured-fields-only: reads name/status/conclusion/started_at/completed_at, never check output or logs. Fail-safe on missing token/head SHA/fetch error.",
633+
},
634+
},
611635
] as const satisfies readonly ReesAnalyzerDoc[];
612636

613637
export const REES_ANALYZER_NAMES = REES_ANALYZERS.map((analyzer) => analyzer.name);

review-enrichment/analyzer-metadata.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,32 @@
688688
"network": "Calls the GitHub PR-reviews API, paginated and bounded to a fixed page cap.",
689689
"notes": "Structured-fields-only: reads state/commit_id/user.login/submitted_at, never diff or review-body text. Fail-safe on missing token/head SHA/fetch error."
690690
}
691+
},
692+
{
693+
"name": "ciCheckSignals",
694+
"title": "CI check-run signals",
695+
"category": "history",
696+
"cost": "github-light",
697+
"defaultEnabled": true,
698+
"profiles": [
699+
"balanced",
700+
"deep"
701+
],
702+
"requires": [
703+
"github-token",
704+
"head-sha"
705+
],
706+
"limits": {
707+
"maxCheckRuns": 100,
708+
"longRunThresholdMinutes": 15
709+
},
710+
"docs": {
711+
"summary": "Flags a named check that only went green after one or more earlier non-success attempts at the current head commit, and any completed check run whose duration crossed a fixed threshold.",
712+
"looksAt": "The head commit's check-runs (one bounded page), grouped by name and ordered by start time.",
713+
"reports": "Check name and either the count of failed attempts before success, or the run's duration in minutes — never logs or output.",
714+
"network": "Calls the GitHub check-runs API once, bounded to one page.",
715+
"notes": "Structured-fields-only: reads name/status/conclusion/started_at/completed_at, never check output or logs. Fail-safe on missing token/head SHA/fetch error."
716+
}
691717
}
692718
]
693719
}
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// CI check-run signals, read from structured GitHub check-run API fields only — no diff/text/log parsing.
2+
// Surfaces two things a PR's own checks tab doesn't summarize at a glance: a named check that only passed after
3+
// one or more earlier non-success attempts at the SAME head commit (retried, not stable-green), and a completed
4+
// check run whose wall-clock duration crossed a fixed threshold (a slow job worth investigating). Reads only
5+
// documented fields from the GitHub check-runs API (name, status, conclusion, started_at, completed_at) and
6+
// compares them — no ambiguous-syntax parsing, so it cannot suffer a patch scanner's edge cases. Pure GitHub-
7+
// metadata read, no repo content, no logs. Fail-safe: no token, no head SHA, a bad repo slug, or a fetch error all
8+
// yield no finding. Bounded to one page of check-runs (MAX_CHECK_RUNS) — realistic CI setups fit comfortably.
9+
import type {
10+
AnalyzerDiagnostics,
11+
CiCheckSignalFinding,
12+
EnrichRequest,
13+
} from "../types.js";
14+
import type { AnalysisContext } from "../analysis-context.js";
15+
import { boundedFetchJson } from "../external-fetch.js";
16+
17+
const GITHUB_API = "https://api.github.com";
18+
const SLUG_RE = /^[A-Za-z0-9._-]+$/;
19+
const MAX_CHECK_RUNS = 100;
20+
const MAX_FINDINGS = 25;
21+
// A completed run at or beyond this wall-clock duration is flagged as long-running.
22+
const LONG_RUN_THRESHOLD_MS = 15 * 60 * 1000;
23+
// Conclusions that count as a non-success ATTEMPT for the retry signal. `neutral`/`skipped` are intentionally
24+
// excluded — neither means the job failed and had to be retried.
25+
const FAILURE_CONCLUSIONS = new Set(["failure", "timed_out", "cancelled", "action_required"]);
26+
27+
interface ScanOptions {
28+
signal?: AbortSignal;
29+
analysis?: Pick<AnalysisContext, "fetchJson">;
30+
diagnostics?: AnalyzerDiagnostics;
31+
}
32+
33+
/** The slice of a GitHub check-run list item this analyzer reads. */
34+
interface CheckRunListItem {
35+
name?: string;
36+
status?: string;
37+
conclusion?: string | null;
38+
started_at?: string;
39+
completed_at?: string | null;
40+
}
41+
42+
interface CheckRunsResponse {
43+
check_runs?: CheckRunListItem[];
44+
}
45+
46+
function githubHeaders(token: string): Record<string, string> {
47+
return {
48+
Authorization: `Bearer ${token}`,
49+
Accept: "application/vnd.github+json",
50+
"X-GitHub-Api-Version": "2022-11-28",
51+
};
52+
}
53+
54+
async function fetchCheckRuns(
55+
owner: string,
56+
repo: string,
57+
headSha: string,
58+
headers: Record<string, string>,
59+
fetchFn: typeof fetch,
60+
signal: AbortSignal | undefined,
61+
options: Pick<ScanOptions, "analysis" | "diagnostics">,
62+
): Promise<CheckRunListItem[] | null> {
63+
const url =
64+
`${GITHUB_API}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits/` +
65+
`${encodeURIComponent(headSha)}/check-runs?per_page=${MAX_CHECK_RUNS}`;
66+
const fetchOptions = {
67+
endpointCategory: "github-check-runs",
68+
headers,
69+
signal,
70+
fetchImpl: fetchFn,
71+
diagnostics: options.diagnostics,
72+
phase: "ci-check-signals",
73+
subcall: "github-check-runs",
74+
maxBytes: 512 * 1024,
75+
};
76+
const response = options.analysis
77+
? await options.analysis.fetchJson<CheckRunsResponse>(url, fetchOptions)
78+
: await boundedFetchJson<CheckRunsResponse>(url, fetchOptions);
79+
return response.ok && Array.isArray(response.data.check_runs) ? response.data.check_runs : null;
80+
}
81+
82+
/** Groups completed runs by name (in the order the API returned them) and, within each group, orders them by
83+
* `started_at` so "the last attempt" and "attempts before it" are well-defined. Runs still in progress/queued
84+
* (no conclusion yet) are excluded — they are not a finished attempt. Pure. */
85+
function groupCompletedRunsByName(runs: CheckRunListItem[]): Map<string, CheckRunListItem[]> {
86+
const groups = new Map<string, CheckRunListItem[]>();
87+
for (const run of runs) {
88+
if (run.status !== "completed" || !run.name || !run.conclusion || !run.started_at) continue;
89+
const list = groups.get(run.name);
90+
if (list) list.push(run);
91+
else groups.set(run.name, [run]);
92+
}
93+
for (const list of groups.values()) {
94+
list.sort((a, b) => (a.started_at! < b.started_at! ? -1 : a.started_at! > b.started_at! ? 1 : 0));
95+
}
96+
return groups;
97+
}
98+
99+
/** Pure reduction: completed check-runs at one head commit → retry/long-run findings, in a stable, bounded order
100+
* (retries first, then long-running runs, each in the order their check name was first seen). */
101+
export function analyzeCheckRuns(runs: CheckRunListItem[]): CiCheckSignalFinding[] {
102+
const findings: CiCheckSignalFinding[] = [];
103+
const groups = groupCompletedRunsByName(runs);
104+
105+
for (const [name, attempts] of groups) {
106+
if (findings.length >= MAX_FINDINGS) break;
107+
const last = attempts[attempts.length - 1]!;
108+
if (last.conclusion === "success") {
109+
const failedAttempts = attempts.slice(0, -1).filter((run) => FAILURE_CONCLUSIONS.has(run.conclusion!)).length;
110+
if (failedAttempts > 0) {
111+
findings.push({ checkName: name, kind: "retried-after-failure", failedAttempts });
112+
}
113+
}
114+
}
115+
116+
for (const [name, attempts] of groups) {
117+
for (const run of attempts) {
118+
if (findings.length >= MAX_FINDINGS) return findings;
119+
if (!run.completed_at) continue;
120+
const durationMs = Date.parse(run.completed_at) - Date.parse(run.started_at!);
121+
if (Number.isFinite(durationMs) && durationMs >= LONG_RUN_THRESHOLD_MS) {
122+
findings.push({
123+
checkName: name,
124+
kind: "long-running-check",
125+
durationMinutes: Math.round(durationMs / 60_000),
126+
});
127+
}
128+
}
129+
}
130+
131+
return findings;
132+
}
133+
134+
/** Analyzer entrypoint: a PR's head-commit check-runs → retry/long-run CI signal findings. Fail-safe — no token,
135+
* no head SHA, a bad repo slug, or a fetch error all yield no finding rather than an error. */
136+
export async function scanCiCheckSignals(
137+
req: EnrichRequest,
138+
fetchFn: typeof fetch = fetch,
139+
options: ScanOptions = {},
140+
): Promise<CiCheckSignalFinding[]> {
141+
const { repoFullName, githubToken, headSha } = req;
142+
if (!githubToken || !headSha) return [];
143+
const parts = repoFullName.split("/");
144+
const owner = parts[0];
145+
const repo = parts[1];
146+
if (parts.length !== 2 || !owner || !repo || !SLUG_RE.test(owner) || !SLUG_RE.test(repo)) return [];
147+
148+
const headers = githubHeaders(githubToken);
149+
const runs = await fetchCheckRuns(owner, repo, headSha, headers, fetchFn, options.signal, options);
150+
if (!runs) return [];
151+
152+
return analyzeCheckRuns(runs);
153+
}

review-enrichment/src/analyzers/registry.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { scanApprovalIntegrity } from "./approval-integrity.js";
33
import { scanAssetWeight } from "./asset-weight.js";
44
import { scanChurnHotspot } from "./churn-hotspot.js";
55
import { scanBlameLink } from "./blame-link.js";
6+
import { scanCiCheckSignals } from "./ci-check-signals.js";
67
import { scanCodeowners } from "./codeowners.js";
78
import { scanCommitSignature } from "./commit-signature.js";
89
import { dependencyAnalyzer } from "./dependency/descriptor.js";
@@ -516,6 +517,41 @@ export const ANALYZER_DESCRIPTORS = [
516517
run: (req, { signal, analysis, diagnostics }) =>
517518
scanApprovalIntegrity(req, fetch, { signal, analysis, diagnostics }),
518519
}),
520+
descriptor({
521+
name: "ciCheckSignals",
522+
title: "CI check-run signals",
523+
category: "history",
524+
cost: "github-light",
525+
defaultEnabled: true,
526+
requires: ["github-token", "head-sha"],
527+
limits: { maxCheckRuns: 100, longRunThresholdMinutes: 15 },
528+
docs: {
529+
summary:
530+
"Flags a named check that only went green after one or more earlier non-success attempts at the current head commit, and any completed check run whose duration crossed a fixed threshold.",
531+
looksAt:
532+
"The head commit's check-runs (one bounded page), grouped by name and ordered by start time.",
533+
reports: "Check name and either the count of failed attempts before success, or the run's duration in minutes — never logs or output.",
534+
network: "Calls the GitHub check-runs API once, bounded to one page.",
535+
notes:
536+
"Structured-fields-only: reads name/status/conclusion/started_at/completed_at, never check output or logs. Fail-safe on missing token/head SHA/fetch error.",
537+
},
538+
render: (findings, helpers) => {
539+
if (!findings.length) return [];
540+
const lines = ["### CI check-run signals"];
541+
for (const item of findings) {
542+
if (item.kind === "retried-after-failure") {
543+
lines.push(
544+
`- ${helpers.safeCodeSpan(item.checkName)} only passed after ${item.failedAttempts} earlier non-success attempt(s) at this commit`,
545+
);
546+
} else {
547+
lines.push(`- ${helpers.safeCodeSpan(item.checkName)} ran for ${item.durationMinutes} minute(s)`);
548+
}
549+
}
550+
return lines;
551+
},
552+
run: (req, { signal, analysis, diagnostics }) =>
553+
scanCiCheckSignals(req, fetch, { signal, analysis, diagnostics }),
554+
}),
519555
] as const satisfies readonly AnyAnalyzerDescriptor[];
520556

521557
export const ANALYZER_NAMES = ANALYZER_DESCRIPTORS.map(

review-enrichment/src/render.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,7 @@ export function renderBrief(
381381

382382
lines.push(...renderDescriptorSection("blameLink", findings.blameLink));
383383
lines.push(...renderDescriptorSection("approvalIntegrity", findings.approvalIntegrity));
384+
lines.push(...renderDescriptorSection("ciCheckSignals", findings.ciCheckSignals));
384385

385386
if (!lines.length) return { promptSection: "", systemSuffix: "" };
386387

review-enrichment/src/types.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,15 @@ export type ApprovalIntegrityFinding =
316316
| { reviewer: string; kind: "self-approval" }
317317
| { reviewer: string; kind: "outstanding-changes-requested" };
318318

319+
/** A CI check-run signal, read from structured GitHub check-run API fields only (name, status, conclusion,
320+
* started_at, completed_at) — never logs or repo content. `retried-after-failure`: the named check's latest
321+
* completed run at the head commit is a success, but one or more earlier completed runs of the SAME name were
322+
* not (failure/timed_out/cancelled/action_required) — it did not go green on the first try.
323+
* `long-running-check`: a single completed run whose wall-clock duration crossed a fixed threshold. */
324+
export type CiCheckSignalFinding =
325+
| { checkName: string; kind: "retried-after-failure"; failedAttempts: number }
326+
| { checkName: string; kind: "long-running-check"; durationMinutes: number };
327+
319328
/** Structured analyzer output. Each analyzer fills its own key; more land as analyzers ship (#1477/#1478). */
320329
export interface BriefFindings {
321330
dependency?: DependencyFinding[];
@@ -341,6 +350,7 @@ export interface BriefFindings {
341350
churnHotspot?: ChurnHotspotFinding[];
342351
blameLink?: BlameLinkFinding[];
343352
approvalIntegrity?: ApprovalIntegrityFinding[];
353+
ciCheckSignals?: CiCheckSignalFinding[];
344354
}
345355

346356
/** A JSDoc/TSDoc block whose `@param` tags name parameters the adjacent function no longer declares — a

review-enrichment/test/analyzer-registry.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const EXPECTED_ANALYZERS = [
3333
"churnHotspot",
3434
"blameLink",
3535
"approvalIntegrity",
36+
"ciCheckSignals",
3637
];
3738

3839
test("analyzer descriptors cover the runtime registry in stable order", () => {

0 commit comments

Comments
 (0)