Skip to content

Commit b8a1ba4

Browse files
authored
fix(review): replace the live in-flight dispatch check with a persisted R2 marker (#4133)
The prior fix queried GitHub's own runs API right before dispatching, which has a real gap: a freshly-dispatched run isn't guaranteed to be visible via that API the instant dispatch returns, so a poll landing in that window could still redispatch and cancel the in-flight run. A persisted marker written synchronously on a successful dispatch, cleared on the webhook_run completion (any conclusion), and self-expiring after the workflow's 15-minute timeout closes that gap without depending on API eventual consistency.
1 parent 06689bc commit b8a1ba4

6 files changed

Lines changed: 263 additions & 112 deletions

File tree

src/queue/processors.ts

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ import { screenshotsAllowed } from "../review/visual-wire";
361361
import { isVisualPath } from "../review/visual/paths";
362362
import { buildCapture, hasSuccessfulBotCapture, resolveVisualRoutes, type CaptureRoute } from "../review/visual/capture";
363363
import {
364+
clearFallbackDispatchMarker,
364365
fallbackShotFileName,
365366
fallbackShotR2Key,
366367
fetchFallbackArtifactShots,
@@ -4601,13 +4602,18 @@ async function maybeCaptureOnActionsFallbackWorkflowRun(
46014602
).workflow_run;
46024603
if (run?.name !== FALLBACK_WORKFLOW_NAME || run?.event !== "workflow_dispatch") return false;
46034604

4604-
if (run.conclusion === "success" && run.id && isConvergenceRepoAllowed(env, repoFullName)) {
4605-
const correlation = parseFallbackRunCorrelation(run.display_title);
4606-
if (correlation) {
4607-
const admissionKey = githubRateLimitAdmissionKeyForInstallation(installationId);
4608-
await storeVisualCaptureFallbackShots(env, repoFullName, installationId, run.id, correlation.prNumber, correlation.headSha, admissionKey);
4609-
await reReviewStoredPullRequest(env, deliveryId, installationId, repoFullName, correlation.prNumber);
4610-
}
4605+
const correlation = parseFallbackRunCorrelation(run.display_title);
4606+
if (correlation) {
4607+
// The run has settled -- success, failure, cancelled, or timed_out all mean "no longer in flight," so
4608+
// clear the dispatch marker regardless of conclusion (#4112 review fix). Otherwise a genuinely failed run
4609+
// would leave the marker in place for the rest of FALLBACK_DISPATCH_MARKER_MAX_AGE_MS, blocking a retry
4610+
// that could otherwise succeed immediately.
4611+
await clearFallbackDispatchMarker(env, correlation.headSha);
4612+
}
4613+
if (run.conclusion === "success" && run.id && correlation && isConvergenceRepoAllowed(env, repoFullName)) {
4614+
const admissionKey = githubRateLimitAdmissionKeyForInstallation(installationId);
4615+
await storeVisualCaptureFallbackShots(env, repoFullName, installationId, run.id, correlation.prNumber, correlation.headSha, admissionKey);
4616+
await reReviewStoredPullRequest(env, deliveryId, installationId, repoFullName, correlation.prNumber);
46114617
}
46124618
await recordWebhookEvent(env, {
46134619
deliveryId,

src/review/visual/actions-fallback.ts

Lines changed: 72 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -144,45 +144,85 @@ export function parseFallbackRunCorrelation(displayTitle: string | undefined | n
144144
return { prNumber, headSha: (match[2] as string).toLowerCase() };
145145
}
146146

147-
/** True when a fallback run for this EXACT (prNumber, headSha) is already queued or in progress -- checked
148-
* by buildCapture before dispatching, so the existing recapture-poll retry (every 90s, up to 5 attempts,
149-
* see PREVIEW_POLL_SECONDS/MAX_PREVIEW_POLLS in processors.ts) doesn't repeatedly re-dispatch while waiting
150-
* for the SAME run's workflow_run completion. That matters because the workflow's own `concurrency: group:
147+
// ---------------------------------------------------------------------------------------------------------
148+
// Dispatch in-flight marker -- a persisted R2 sentinel, not a live GitHub API query (#4112 review fix).
149+
// ---------------------------------------------------------------------------------------------------------
150+
151+
const FALLBACK_DISPATCH_MARKER_NAMESPACE = "gittensory/fallback-dispatch/";
152+
153+
/** The workflow's own `timeout-minutes: 15` (visual-capture-fallback.yml) plus a buffer for GitHub's own
154+
* runner-queueing delay before the job even starts -- a marker older than this is treated as abandoned
155+
* (the run either finished without a webhook ever reaching us, or GitHub silently dropped the dispatch)
156+
* rather than blocking dispatch forever. */
157+
const FALLBACK_DISPATCH_MARKER_MAX_AGE_MS = 18 * 60 * 1000;
158+
159+
async function fallbackDispatchMarkerR2Key(headSha: string): Promise<string> {
160+
const fingerprint = await sha256Hex(`${headSha}:actions-fallback:dispatch-marker`);
161+
return `${FALLBACK_DISPATCH_MARKER_NAMESPACE}${fingerprint.slice(0, 40)}.json`;
162+
}
163+
164+
/** True when a fallback run for this head SHA was dispatched recently enough that it may still be
165+
* queued/in-progress -- checked by buildCapture BEFORE dispatching, so the existing recapture-poll retry
166+
* (every 90s, up to 5 attempts -- see PREVIEW_POLL_SECONDS/MAX_PREVIEW_POLLS in processors.ts, a 7.5-minute
167+
* window comfortably inside the workflow's own 15-minute timeout) doesn't repeatedly re-dispatch while a
168+
* build is still running. That matters because the workflow's own `concurrency: group:
151169
* visual-capture-fallback-${{ inputs.head_sha }}` + `cancel-in-progress: true` means a second dispatch for
152-
* the same head SHA CANCELS the first -- without this check, a poll firing before a slow build finishes
153-
* would cancel-and-restart it every 90s and the fallback could never complete. Queries GitHub's own run
154-
* list rather than persisting new dispatch-tracking state, mirroring this pipeline's existing
155-
* live-query-don't-persist pattern (getLatestDeploymentStatus, findPreviewUrlFromChecks). Fails OPEN (false,
156-
* "nothing in flight") on any error -- a transient list-runs failure should still let the existing
157-
* concurrency group be the backstop dedup, not silently stop the fallback from ever being tried. */
158-
export async function hasInFlightFallbackDispatch(params: {
159-
token: string;
160-
repo: GitHubRepo;
161-
prNumber: number;
162-
headSha: string;
163-
rateLimitAdmissionKey?: GitHubRateLimitAdmissionKey | undefined;
164-
}): Promise<boolean> {
165-
const base = `https://api.github.com/repos/${params.repo.owner}/${params.repo.repo}`;
170+
* the same head SHA CANCELS the first -- without this check, a poll firing well within the 15-minute budget
171+
* would cancel-and-restart the run on every single poll and the fallback could never complete.
172+
*
173+
* A PERSISTED marker (not a live GitHub list-runs query) is deliberate: a freshly-dispatched run isn't
174+
* guaranteed to be visible via the Actions API the instant `dispatchVisualCaptureFallback` returns (GitHub's
175+
* own eventual consistency), so a live query taken right after dispatch could itself race and report
176+
* "nothing in flight" moments after a dispatch just succeeded. Writing the marker synchronously on a
177+
* successful dispatch closes that gap. Fails OPEN (false, "nothing in flight") on any read error -- a
178+
* transient R2 failure should still let the existing concurrency group be the backstop dedup, not silently
179+
* stop the fallback from ever being tried. */
180+
export async function isFallbackDispatchInFlight(env: Env, headSha: string): Promise<boolean> {
181+
if (!env.REVIEW_AUDIT) return false;
166182
try {
167-
const response = await timeoutFetch(`${base}/actions/workflows/${FALLBACK_WORKFLOW_FILE}/runs?event=workflow_dispatch&per_page=20`, {
168-
headers: githubApiHeaders(params.token),
169-
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
170-
githubRateLimitAdmission: params.rateLimitAdmissionKey !== undefined,
171-
...(params.rateLimitAdmissionKey ? { githubRateLimitAdmissionKey: params.rateLimitAdmissionKey } : {}),
172-
});
173-
if (!response.ok) return false;
174-
const payload = (await response.json().catch(() => null)) as { workflow_runs?: Array<{ status?: string; display_title?: string }> } | null;
175-
const headSha = params.headSha.toLowerCase();
176-
return (payload?.workflow_runs ?? []).some((run) => {
177-
if (run.status !== "queued" && run.status !== "in_progress") return false;
178-
const correlation = parseFallbackRunCorrelation(run.display_title);
179-
return correlation !== null && correlation.prNumber === params.prNumber && correlation.headSha === headSha;
180-
});
183+
const object = await env.REVIEW_AUDIT.get(await fallbackDispatchMarkerR2Key(headSha));
184+
if (!object) return false;
185+
const text = await new Response(object.body).text();
186+
const marker = JSON.parse(text) as { dispatchedAt?: number };
187+
if (typeof marker.dispatchedAt !== "number") return false;
188+
return Date.now() - marker.dispatchedAt < FALLBACK_DISPATCH_MARKER_MAX_AGE_MS;
181189
} catch {
182190
return false;
183191
}
184192
}
185193

194+
/** Record that a fallback dispatch just succeeded for this head SHA, so a subsequent buildCapture call
195+
* (e.g. the next recapture poll) sees it via isFallbackDispatchInFlight instead of re-dispatching. Best
196+
* effort -- a failed write just means the concurrency group's cancel-in-progress behavior is the only
197+
* remaining backstop, same as before this marker existed. */
198+
export async function markFallbackDispatched(env: Env, headSha: string): Promise<void> {
199+
if (!env.REVIEW_AUDIT) return;
200+
try {
201+
const key = await fallbackDispatchMarkerR2Key(headSha);
202+
await env.REVIEW_AUDIT.put(key, JSON.stringify({ dispatchedAt: Date.now() }), {
203+
httpMetadata: { contentType: "application/json" },
204+
});
205+
} catch {
206+
// best effort -- see doc comment above
207+
}
208+
}
209+
210+
/** Clear the in-flight marker once the dispatched run has settled (ANY conclusion -- success, failure,
211+
* cancelled, timed_out all mean "no longer in flight"), called from the workflow_run webhook handler in
212+
* processors.ts. Best effort -- if this never runs (a lost webhook delivery), FALLBACK_DISPATCH_MARKER_MAX_AGE_MS
213+
* is the fail-safe expiry so a genuinely stuck marker can't block retries forever. A try/catch (not just a
214+
* `.catch()` on the delete call) matters here: a minimal/partial R2Bucket implementation that doesn't
215+
* implement `delete` at all throws SYNCHRONOUSLY at the call site (`TypeError: ... is not a function`),
216+
* before any `.catch()` on its return value would even attach. */
217+
export async function clearFallbackDispatchMarker(env: Env, headSha: string): Promise<void> {
218+
if (!env.REVIEW_AUDIT) return;
219+
try {
220+
await env.REVIEW_AUDIT.delete(await fallbackDispatchMarkerR2Key(headSha));
221+
} catch {
222+
// best effort -- see doc comment above
223+
}
224+
}
225+
186226
// ---------------------------------------------------------------------------------------------------------
187227
// Minimal ZIP reader -- just enough to read a GitHub Actions artifact (STORED / DEFLATE entries only).
188228
// ---------------------------------------------------------------------------------------------------------

src/review/visual/capture.ts

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// default TanStack route convention; those hooks can return if a per-repo visual config is added.
1313
import { sha256Hex } from "../../utils/crypto";
1414
import type { GitHubRateLimitAdmissionKey } from "../../github/client";
15-
import { dispatchVisualCaptureFallback, fallbackShotR2Key, hasInFlightFallbackDispatch } from "./actions-fallback";
15+
import { dispatchVisualCaptureFallback, fallbackShotR2Key, isFallbackDispatchInFlight, markFallbackDispatched } from "./actions-fallback";
1616
import {
1717
findPreviewUrlFromChecks,
1818
findPreviewUrlFromPrComments,
@@ -408,20 +408,25 @@ export async function buildCapture(env: Env, token: string, target: CaptureTarge
408408
// Never re-dispatch onto an already in-flight run (#4112 review fix): the workflow's own `concurrency:
409409
// cancel-in-progress: true` group would CANCEL that run the instant a second dispatch for the same head
410410
// SHA lands, so a recapture-poll retry (every 90s -- see PREVIEW_POLL_SECONDS in processors.ts) firing
411-
// before a slower build finishes could cancel-and-restart it forever and never complete. See
412-
// hasInFlightFallbackDispatch's own doc comment for the full rationale.
413-
const alreadyInFlight = await hasInFlightFallbackDispatch({ token, repo, prNumber: target.prNumber, headSha: target.headSha, rateLimitAdmissionKey });
414-
const dispatched =
415-
alreadyInFlight ||
416-
(await dispatchVisualCaptureFallback({
411+
// well within the workflow's 15-minute timeout could cancel-and-restart it on every poll and never
412+
// complete. isFallbackDispatchInFlight checks a PERSISTED R2 marker rather than querying GitHub's runs
413+
// API live, so there's no eventual-consistency gap right after a dispatch just succeeded -- see its own
414+
// doc comment for the full rationale. markFallbackDispatched writes that marker on a successful dispatch;
415+
// the webhook handler (processors.ts) clears it once the run settles.
416+
const alreadyInFlight = await isFallbackDispatchInFlight(env, target.headSha);
417+
let dispatched = alreadyInFlight;
418+
if (!dispatched) {
419+
dispatched = await dispatchVisualCaptureFallback({
417420
token,
418421
repo,
419422
ref: target.defaultBranchRef,
420423
prNumber: target.prNumber,
421424
headSha: target.headSha,
422425
routes,
423426
rateLimitAdmissionKey,
424-
}));
427+
});
428+
if (dispatched) await markFallbackDispatched(env, target.headSha);
429+
}
425430
if (dispatched) previewPending = true;
426431
}
427432

test/unit/actions-fallback-webhook.test.ts

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
} from "../../src/db/repositories";
1111
import { clearInstallationTokenCacheForTest } from "../../src/github/app";
1212
import { clearGitHubResponseCacheForTest } from "../../src/github/client";
13-
import { fallbackShotR2Key, FALLBACK_ARTIFACT_NAME } from "../../src/review/visual/actions-fallback";
13+
import { fallbackShotR2Key, FALLBACK_ARTIFACT_NAME, isFallbackDispatchInFlight, markFallbackDispatched } from "../../src/review/visual/actions-fallback";
1414
import { processJob } from "../../src/queue/processors";
1515
import { createTestEnv } from "../helpers/d1";
1616

@@ -91,6 +91,9 @@ function memoryReviewAudit(): R2Bucket {
9191
store.set(key, bytes);
9292
return { key } as unknown as R2Object;
9393
},
94+
async delete(key: string) {
95+
store.delete(key);
96+
},
9497
} as unknown as R2Bucket;
9598
}
9699

@@ -481,6 +484,72 @@ describe("workflow_run webhook -> actions_fallback storage (#4112)", () => {
481484
expect(artifactsListCalled).toBe(false);
482485
});
483486

487+
it("clears the dispatch marker on a FAILED run too (#4112 review fix -- a failed run shouldn't block a retry for the rest of the max-age window)", async () => {
488+
const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_REPOS: "owner/fallback-repo", REVIEW_AUDIT: memoryReviewAudit() });
489+
await seedRepoAndPr(env, "cafebabecafebabecafebabecafebabecafebabe");
490+
await markFallbackDispatched(env, "cafebabecafebabecafebabecafebabecafebabe");
491+
await expect(isFallbackDispatchInFlight(env, "cafebabecafebabecafebabecafebabecafebabe")).resolves.toBe(true);
492+
vi.stubGlobal("fetch", baseFetchStub({}));
493+
494+
await processJob(env, {
495+
type: "github-webhook",
496+
deliveryId: "failed-run-clears-marker",
497+
eventName: "workflow_run",
498+
payload: {
499+
action: "completed",
500+
repository: { name: "fallback-repo", full_name: "owner/fallback-repo", owner: { login: "owner" } },
501+
installation: { id: 9101 },
502+
workflow_run: { id: 590, name: "Gittensory Visual Capture Fallback", event: "workflow_dispatch", conclusion: "failure", display_title: "gittensory-visual-fallback pr=55 sha=cafebabecafebabecafebabecafebabecafebabe" },
503+
},
504+
} as never);
505+
506+
await expect(isFallbackDispatchInFlight(env, "cafebabecafebabecafebabecafebabecafebabe")).resolves.toBe(false);
507+
});
508+
509+
it("clears the dispatch marker on a SUCCESSFUL run as well", async () => {
510+
const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_REPOS: "owner/fallback-repo", REVIEW_AUDIT: memoryReviewAudit() });
511+
await seedRepoAndPr(env, "cafebabecafebabecafebabecafebabecafebabe");
512+
await markFallbackDispatched(env, "cafebabecafebabecafebabecafebabecafebabe");
513+
vi.stubGlobal("fetch", baseFetchStub({ "/actions/runs/": () => Response.json({ artifacts: [] }) }));
514+
515+
await processJob(env, {
516+
type: "github-webhook",
517+
deliveryId: "success-run-clears-marker",
518+
eventName: "workflow_run",
519+
payload: {
520+
action: "completed",
521+
repository: { name: "fallback-repo", full_name: "owner/fallback-repo", owner: { login: "owner" } },
522+
installation: { id: 9101 },
523+
workflow_run: { id: 591, name: "Gittensory Visual Capture Fallback", event: "workflow_dispatch", conclusion: "success", display_title: "gittensory-visual-fallback pr=55 sha=cafebabecafebabecafebabecafebabecafebabe" },
524+
},
525+
} as never);
526+
527+
await expect(isFallbackDispatchInFlight(env, "cafebabecafebabecafebabecafebabecafebabe")).resolves.toBe(false);
528+
});
529+
530+
it("does not clear any marker when the run's display_title doesn't correlate to a PR (nothing to key the clear on)", async () => {
531+
const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_REPOS: "owner/fallback-repo", REVIEW_AUDIT: memoryReviewAudit() });
532+
await seedRepoAndPr(env, "cafebabecafebabecafebabecafebabecafebabe");
533+
await markFallbackDispatched(env, "cafebabecafebabecafebabecafebabecafebabe");
534+
vi.stubGlobal("fetch", baseFetchStub({}));
535+
536+
await processJob(env, {
537+
type: "github-webhook",
538+
deliveryId: "uncorrelated-run-leaves-marker",
539+
eventName: "workflow_run",
540+
payload: {
541+
action: "completed",
542+
repository: { name: "fallback-repo", full_name: "owner/fallback-repo", owner: { login: "owner" } },
543+
installation: { id: 9101 },
544+
workflow_run: { id: 592, name: "Gittensory Visual Capture Fallback", event: "workflow_dispatch", conclusion: "success", display_title: "manually triggered" },
545+
},
546+
} as never);
547+
548+
// The marker is keyed by headSha "cafebabe...", which this run's uncorrelated title can't recover --
549+
// it must stay untouched (still in flight) rather than being guessed/cleared.
550+
await expect(isFallbackDispatchInFlight(env, "cafebabecafebabecafebabecafebabecafebabe")).resolves.toBe(true);
551+
});
552+
484553
it("does nothing when the run's display_title doesn't correlate to a PR (never guesses)", async () => {
485554
const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_REPOS: "owner/fallback-repo", REVIEW_AUDIT: memoryReviewAudit() });
486555
await seedRepoAndPr(env, "cafebabecafebabecafebabecafebabecafebabe");

0 commit comments

Comments
 (0)