Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/env.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,12 @@ declare global {
* for just that repo. Default OFF — unset/false means the cron tick enqueues NO watchdog job (does no new
* work), so the worker is byte-identical to today. */
GITTENSORY_SWEEP_WATCHDOG?: string;
/** Self-heal: when truthy, a short-interval cron list-diffs GitHub's open PR numbers against the local
* table for every acting-autonomy repo and catches up (fetch + upsert + regate) any PR number GitHub has
* that the local table doesn't — a silently-lost "opened" webhook, caught within minutes instead of the
* 6-hour backfillRegisteredRepositories freshness window. Default OFF — unset/false means the cron tick
* enqueues NO reconciliation job, so the worker is byte-identical to today. */
GITTENSORY_PR_RECONCILIATION?: string;
/** Convergence (RAG retrieval): when truthy, the AI reviewer prompt gains a RELEVANT EXISTING CODE / DOCS
* section — at review time the codebase vector index is queried for code/docs semantically related to the
* PR's changed files (callers, related modules, existing conventions) and appended as additive reference
Expand Down
46 changes: 46 additions & 0 deletions src/github/backfill.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3132,6 +3132,52 @@ export async function fetchLivePullRequest(
return result.status === "ok" ? result.data : undefined;
}

// Bounded at 1000 open PRs (10 pages of 100) — far beyond any real repo's open-PR count; a pathological repo
// can't spin this loop forever.
const RECONCILE_OPEN_PRS_MAX_PAGES = 10;

export type OpenPrReconciliationResult = {
repoFullName: string;
remoteOpenCount: number;
localOpenCount: number;
/** PR numbers GitHub reports open that have NO local pull_requests row at all — a webhook silently lost. */
missingNumbers: number[];
};

/**
* Fast open-PR reconciliation (#audit-open-pr-reconciliation): a CHEAP list-only GitHub read (PR numbers only,
* via the Link-header-paginated `/pulls?state=open` list — never a per-PR detail fetch) diffed against the
* local `pull_requests` table. Exists to catch a silently-lost "PR opened" webhook (#3782/#3793, 2026-07-06:
* two contributor PRs had zero trace anywhere for ~2 hours) within minutes, instead of waiting on
* `backfillRegisteredRepositories`'s 6-hour freshness window.
*
* FAIL-OPEN: a total read failure (repo not found, or the very first list page erroring) returns an all-zero,
* empty-`missingNumbers` result — it must never falsely report every local PR as "missing" just because GitHub
* was briefly unreachable. A LATER page failing mid-crawl keeps the pages already fetched (a partial remote
* list can only under-report divergence, never wrongly flag a real local PR as missing).
*/
export async function reconcileOpenPullRequests(env: Env, repoFullName: string): Promise<OpenPrReconciliationResult> {
const empty: OpenPrReconciliationResult = { repoFullName, remoteOpenCount: 0, localOpenCount: 0, missingNumbers: [] };
const repo = await getRepository(env, repoFullName);
if (!repo) return empty;
const token = await tokenForRepo(env, repo);
const admissionKey = repoAdmissionKeyForToken(env, repo, token);
const remoteNumbers: number[] = [];
for (let page = 1; page <= RECONCILE_OPEN_PRS_MAX_PAGES; page += 1) {
const result = await githubJsonWithHeaders<Array<{ number: number }>>(env, repoFullName, `/pulls?state=open&per_page=100&page=${page}`, token, githubRateLimitOptions(admissionKey)).catch(() => undefined);
if (!result) {
if (page === 1) return empty;
break;
}
remoteNumbers.push(...result.data.map((pr) => pr.number));
if (!hasNextPage(result.link)) break;
}
const localOpen = await listOpenPullRequests(env, repoFullName);
const localNumbers = new Set(localOpen.map((pr) => pr.number));
const missingNumbers = remoteNumbers.filter((number) => !localNumbers.has(number));
return { repoFullName, remoteOpenCount: remoteNumbers.length, localOpenCount: localOpen.length, missingNumbers };
}

// #2537: durable, webhook-invalidated cache for the bare PR-state read (GET /pulls/{n}). Unlike the request-local
// LiveGithubFacts memo (queue/processors.ts), this survives ACROSS webhook deliveries / sweep ticks, cutting
// repeat /pulls/{n} calls for an unchanged PR at the freshness-guard/readiness/dup-winner call sites. NEVER used
Expand Down
9 changes: 9 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { processJob } from "./queue/processors";
import { isOrbBrokerEnabled } from "./orb/broker";
import { isOpsEnabled } from "./review/ops-wire";
import { isSweepWatchdogEnabled } from "./review/sweep-watchdog";
import { isPrReconciliationEnabled } from "./review/pr-reconciliation";
import { isRagEnabled } from "./review/rag-wire";
import { isSelfTuneEnabled } from "./review/selftune-wire";
import {
Expand Down Expand Up @@ -106,6 +107,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
const hour = scheduledAt.getUTCHours();
const isHourly = minute === 0;
const isFullSyncWindow = isHourly && hour % 6 === 0;
// Self-heal (#audit-open-pr-reconciliation): every 10 minutes — much tighter than the hourly ops-alerts/
// selftune cadence, since this exists specifically to catch a silently-lost webhook within minutes rather
// than up to 6 hours (backfillRegisteredRepositories's freshness window).
const isReconciliationWindow = minute % 10 === 0;
// The light auto-maintain sweep runs EVERY cron tick (~every 2 min) so an approved+clean PR MERGES and a
// red-CI non-owner PR CLOSES promptly — reviewbot parity (its cron fired every minute). It re-fetches LIVE CI +
// mergeable and only ACTS (merge/close/hold); it never re-runs the AI, so it is cheap enough for this cadence.
Expand Down Expand Up @@ -171,6 +176,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
// classified (MAINTENANCE_JOB_TYPES) so it defers under live-work pressure like every other periodic sweep here.
if (selfHostedReviews) jobs.push({ type: "backlog-convergence-sweep", requestedBy: "schedule" });
}
// Self-heal (flag GITTENSORY_PR_RECONCILIATION). Every 10 minutes — see isReconciliationWindow above.
// Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never created, so the cron tick does
// ZERO new work and the enqueued set is byte-identical to today.
if (selfHostedReviews && isReconciliationWindow && isPrReconciliationEnabled(env)) jobs.push({ type: "reconcile-open-prs", requestedBy: "schedule" });
if (isHourly) {
jobs.push({ type: "refresh-registry", requestedBy: "schedule" });
jobs.push({ type: "refresh-scoring-model", requestedBy: "schedule" });
Expand Down
7 changes: 7 additions & 0 deletions src/queue/processors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,7 @@ import { resolveUnlinkedIssueMatchDisposition } from "../review/unlinked-issue-g
import { DEFAULT_SCREENSHOT_TABLE_GATE, evaluateScreenshotTableGate } from "../review/screenshot-table-gate";
import { isOpsEnabled, runOpsAlerts } from "../review/ops-wire";
import { isSweepWatchdogEnabled, runSweepLivenessWatchdog } from "../review/sweep-watchdog";
import { isPrReconciliationEnabled, runOpenPrReconciliation } from "../review/pr-reconciliation";
import { isSelfTuneEnabled, runSelfTune } from "../review/selftune-wire";
import {
isCloseHoldOnly,
Expand Down Expand Up @@ -1119,6 +1120,12 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
// work here too. Fails safe internally — never throws into the queue.
if (isSweepWatchdogEnabled(env)) await runSweepLivenessWatchdog(env);
return;
case "reconcile-open-prs":
// Self-heal (flag GITTENSORY_PR_RECONCILIATION). Defense-in-depth: the cron only ENQUEUES this when the
// flag is ON, but a stale in-flight job that lands after a flag-flip must still no-op, so flag-OFF does
// zero work here too. Fails safe internally — never throws into the queue.
if (isPrReconciliationEnabled(env)) await runOpenPrReconciliation(env);
return;
case "selftune":
// Convergence (self-improve / auto-tune, flag GITTENSORY_REVIEW_SELFTUNE). Defense-in-depth: the cron only
// ENQUEUES this when the flag is ON, but a stale in-flight job that lands after a flag-flip must still
Expand Down
124 changes: 124 additions & 0 deletions src/review/pr-reconciliation.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Self-heal (flag-gated by GITTENSORY_PR_RECONCILIATION). A "PR opened" webhook that silently vanishes (no
// error, no audit event, no trace anywhere — see reconcileOpenPullRequests's doc comment for the 2026-07-06
// incident that motivated this) previously wasn't caught until backfillRegisteredRepositories's opportunistic
// resync, which skips any repo whose sync is "fresh" (up to 6 hours). This module runs a much tighter, dedicated
// reconciliation on its own short cron cadence: list-diff GitHub's open PR numbers against the local table for
// every acting-autonomy repo, and for any number GitHub has that the local table doesn't, immediately catch it
// up (fetch full details, upsert, and enqueue a normal regate — the SAME pipeline a real webhook would have fed).
//
// Default OFF (like every other convergence capability) — flag-OFF this module is never invoked and the cron
// enqueues no reconciliation job, byte-identical to today.

import { githubRateLimitAdmissionKeyForToken } from "../github/client";
import { createInstallationToken } from "../github/app";
import { fetchLivePullRequest, reconcileOpenPullRequests } from "../github/backfill";
import { getRepository, listRepositories, upsertPullRequestFromGitHub } from "../db/repositories";
import { isAgentConfigured } from "../settings/autonomy";
import { resolveRepositorySettings } from "../settings/repository-settings";
import { incr } from "../selfhost/metrics";
import type { JobMessage } from "../types";
import { errorMessage } from "../utils/json";
import { isConvergenceRepoAllowed, listConvergenceRepos } from "./cutover-gate";

/** True when fast open-PR reconciliation is enabled. Flag-OFF (default) → the caller never invokes it, so the
* cron enqueues no reconciliation job and the queue processor no-ops on a stale in-flight one. */
export function isPrReconciliationEnabled(env: { GITTENSORY_PR_RECONCILIATION?: string | undefined }): boolean {
return /^(1|true|yes|on)$/i.test(env.GITTENSORY_PR_RECONCILIATION ?? "");
}

/** The same acting-autonomy repo set fanOutAgentRegateSweepJobs sweeps (mirrors sweep-watchdog.ts's own copy of
* this selection) — this reconciliation only makes sense for repos gittensory is actually reviewing. */
async function watchedRepos(env: Env): Promise<Array<{ fullName: string; installationId?: number }>> {
const repositoriesByKey = new Map((await listRepositories(env)).map((repo) => [repo.fullName.toLowerCase(), repo]));
const byKey = new Map<string, { fullName: string; installationId?: number }>();
for (const repo of repositoriesByKey.values())
byKey.set(repo.fullName.toLowerCase(), { fullName: repo.fullName, ...(typeof repo.installationId === "number" ? { installationId: repo.installationId } : {}) });
for (const fullName of listConvergenceRepos(env)) {
const repo = repositoriesByKey.get(fullName.toLowerCase());
byKey.set(fullName.toLowerCase(), {
fullName,
...(typeof repo?.installationId === "number" ? { installationId: repo.installationId } : {}),
});
}
const configured: Array<{ fullName: string; installationId?: number }> = [];
for (const repo of byKey.values()) {
try {
const settings = await resolveRepositorySettings(env, repo.fullName);
if (isConvergenceRepoAllowed(env, repo.fullName) || isAgentConfigured(settings.autonomy)) configured.push(repo);
} catch {
/* a settings blip on one repo must not abort the whole reconciliation scan */
}
}
return configured;
}

/** Catch up ONE missing PR number: fetch its full live payload, upsert it into the local table, and enqueue a
* normal regate — the exact pipeline a real "PR opened" webhook would have fed it into, so it gets reviewed,
* labeled, and gated exactly like any other PR. Best-effort: a fetch/upsert failure is logged and skipped (the
* next reconciliation tick retries it; it is not lost by this catch-up failing). */
async function catchUpMissingPullRequest(env: Env, repoFullName: string, installationId: number, prNumber: number): Promise<void> {
try {
const token = (await createInstallationToken(env, installationId).catch(() => undefined)) ?? env.GITHUB_PUBLIC_TOKEN;
const admissionKey = githubRateLimitAdmissionKeyForToken(env, token, installationId);
const live = await fetchLivePullRequest(env, repoFullName, prNumber, token, admissionKey);
if (!live) {
console.error(JSON.stringify({ level: "error", event: "open_pr_reconciliation_catch_up_fetch_failed", repository: repoFullName, prNumber }));
return;
}
await upsertPullRequestFromGitHub(env, repoFullName, live);
const message: JobMessage = { type: "agent-regate-pr", deliveryId: `reconcile:${repoFullName}#${prNumber}`, repoFullName, prNumber, installationId };
await env.JOBS.send(message);
} catch (error) {
console.error(JSON.stringify({ level: "error", event: "open_pr_reconciliation_catch_up_failed", repository: repoFullName, prNumber, message: errorMessage(error).slice(0, 200) }));
}
}

export interface OpenPrDivergence {
repoFullName: string;
remoteOpenCount: number;
localOpenCount: number;
missingNumbers: number[];
}

/**
* The reconciliation scan, run on the cron tick. FAILS SAFE: a per-repo error is logged and the scan continues;
* a top-level error is swallowed (this is best-effort self-heal, never a reason to fail the queue). Only an
* INSTALLED repo (installationId present) is reconciled — a registered-but-uninstalled repo never gets a per-PR
* fan-out regardless (#sweep-uninstalled-budget-waste), so reconciling it would just spend the shared
* GITHUB_PUBLIC_TOKEN budget with no actionable outcome. Returns the divergences found (for tests / a caller
* that wants to inspect them further).
*
* Caller MUST gate this on {@link isPrReconciliationEnabled} — it is invoked only from the flag-ON cron path, so
* flag-OFF this function is never reached and the cron does zero new work.
*/
export async function runOpenPrReconciliation(env: Env): Promise<OpenPrDivergence[]> {
const found: OpenPrDivergence[] = [];
try {
const repos = await watchedRepos(env);
for (const repo of repos) {
try {
if (typeof repo.installationId !== "number") continue;
const result = await reconcileOpenPullRequests(env, repo.fullName);
if (result.missingNumbers.length === 0) continue;
found.push({ repoFullName: repo.fullName, remoteOpenCount: result.remoteOpenCount, localOpenCount: result.localOpenCount, missingNumbers: result.missingNumbers });
incr("gittensory_open_pr_reconciliation_missing_total", { repo: repo.fullName }, result.missingNumbers.length);
console.error(
JSON.stringify({
level: "error",
event: "open_pr_reconciliation_divergence",
repository: repo.fullName,
remoteOpenCount: result.remoteOpenCount,
localOpenCount: result.localOpenCount,
missingNumbers: result.missingNumbers,
}),
);
for (const prNumber of result.missingNumbers) await catchUpMissingPullRequest(env, repo.fullName, repo.installationId, prNumber);
} catch (error) {
console.error(JSON.stringify({ level: "error", event: "open_pr_reconciliation_repo_error", repository: repo.fullName, message: errorMessage(error).slice(0, 200) }));
}
}
} catch (error) {
console.error(JSON.stringify({ level: "error", event: "open_pr_reconciliation_error", message: errorMessage(error).slice(0, 200) }));
}
return found;
}
1 change: 1 addition & 0 deletions src/selfhost/maintenance-admission.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ export const MAINTENANCE_JOB_TYPES: ReadonlySet<string> = new Set([
"notify-deliver",
"ops-alerts",
"sweep-liveness-watchdog",
"reconcile-open-prs",
"selftune",
"rag-index-repo",
"backlog-convergence-sweep",
Expand Down
9 changes: 9 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,15 @@ export type JobMessage =
type: "sweep-liveness-watchdog";
requestedBy: "schedule" | "api" | "test";
}
| {
// Self-heal (flag-gated by GITTENSORY_PR_RECONCILIATION). List-diff GitHub's open PR numbers against the
// local table for every acting-autonomy repo — a much tighter cadence than backfillRegisteredRepositories's
// 6-hour freshness window — and catch up (fetch + upsert + regate) any PR number GitHub has that the local
// table doesn't (a silently-lost "opened" webhook). Enqueued on a short interval ONLY when the flag is ON
// (index.ts), so flag-OFF this job never exists.
type: "reconcile-open-prs";
requestedBy: "schedule" | "api" | "test";
}
| {
// Convergence (self-improve / auto-tune, flag-gated by GITTENSORY_REVIEW_SELFTUNE). Run the ported
// self-improvement loop over gittensory's review-outcome data — compute tuning recommendations,
Expand Down
Loading
Loading