Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion src/selfhost/sentry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,21 @@ const SENTRY_MONITORS: Record<SentryMonitorName, { slug: string; config: SentryM
slug: "orb-relay-drain",
config: {
schedule: { type: "interval", value: 1, unit: "minute" },
checkinMargin: 2,
// #6685-followup-2 (LOOPOVER-12, 1600+ "missed check-in" occurrences over 2+ weeks, still recurring):
// checkinMargin: 2 was too tight for a self-host container that gets fully recreated on every redeploy,
// not just restarted in place -- both leading in-process hypotheses were investigated and ruled out with
// real production evidence: the reentrancy-guard overlap-skip added in #instrument-drain-skip (see
// withOrbRelayDrainReentrancyGuard's own header comment) never fired once in 14 days of live coverage
// (loopover_orb_relay_drain_skipped_total stayed at 0), and the genuine in-process timeout errors
// (LOOPOVER-1Y/1Q, kind: orb_relay_drain) total only ~13 occurrences -- nowhere near enough to explain
// 1600+ missed check-ins. What DOES correlate: this container's own StartedAt resets on every redeploy
// (unlike its sidecars', which stay up for days), and boot involves an image pull + up to a 30s Postgres-
// readiness retry loop (waitForPostgres, server.ts) + app init BEFORE this drain loop's first tick can
// even register -- routinely exceeding the old 2-minute margin on a fleet that redeploys multiple times a
// day. 6 minutes gives a normal single redeploy cycle room to complete without a false "outage" issue,
// while a genuinely stuck/crash-looping instance still alerts within ~3 x (1+6) = ~21 minutes
// (failureIssueThreshold below is unchanged -- only the margin, which is what the evidence points at).
checkinMargin: 6,
maxRuntime: 1,
failureIssueThreshold: 3,
recoveryThreshold: 1,
Expand Down
24 changes: 24 additions & 0 deletions test/unit/selfhost-sentry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,30 @@ describe("enabled when SENTRY_DSN is set", () => {
expect(mocks.captureException).not.toHaveBeenCalled();
});

it("gives orb-relay-drain a 6-minute checkin margin so a normal redeploy cycle doesn't false-positive as a missed check-in (LOOPOVER-12 regression)", async () => {
await initSentry({
SENTRY_DSN: "d",
SENTRY_ENVIRONMENT: "prod",
} as unknown as NodeJS.ProcessEnv);

await expect(
withSentryMonitor("orb-relay-drain", { jobType: "orb-relay-drain" }, async () => "ok"),
).resolves.toBe("ok");

expect(mocks.captureCheckIn).toHaveBeenNthCalledWith(
1,
{ monitorSlug: "loopover-selfhost-prod-orb-relay-drain", status: "in_progress" },
expect.objectContaining({
schedule: { type: "interval", value: 1, unit: "minute" },
checkinMargin: 6,
maxRuntime: 1,
failureIssueThreshold: 3,
recoveryThreshold: 1,
}),
);
expect(mocks.captureException).not.toHaveBeenCalled();
});

it("records failed Sentry cron monitor check-ins with sanitized context", async () => {
await initSentry({
SENTRY_DSN: "d",
Expand Down
Loading