diff --git a/packages/core/src/types/sync/health.contracts.ts b/packages/core/src/types/sync/health.contracts.ts index b533385a7..26534daec 100644 --- a/packages/core/src/types/sync/health.contracts.ts +++ b/packages/core/src/types/sync/health.contracts.ts @@ -61,3 +61,23 @@ export const SyncHealthSnapshotSchema = z.strictObject({ export type SyncHealthSnapshot = z.infer; export const SYNC_HEALTH_SNAPSHOT_EVENT = "sync_health_snapshot" as const; + +// Emitted once per reconcile-sweep cycle completion (~every 10 min), rather +// than on a fixed telemetry timer like the health snapshot above. A queue +// depth GAUGE sampled every 5 minutes cannot reliably detect a sweep that +// enqueues and fully drains work in well under a minute — an alert built on +// jobs.pending fired repeatedly on a healthy fleet because the 5-minute +// sampler almost never caught the queue non-empty (2026-08-01). Counting +// discrete sweep-completion events instead sidesteps the sampling gap +// entirely: a sweep either ran or it didn't, with nothing in between for a +// sampler to miss. +export const SyncReconcileSweepEventSchema = z.strictObject({ + environment: z.string().min(1), + service: z.literal("compass-sync"), + enqueued: z.number().int().nonnegative(), +}); +export type SyncReconcileSweepEvent = z.infer< + typeof SyncReconcileSweepEventSchema +>; + +export const SYNC_RECONCILE_SWEEP_EVENT = "sync_reconcile_sweep" as const; diff --git a/packages/sync/src/app.ts b/packages/sync/src/app.ts index dbe491760..13f7693e5 100644 --- a/packages/sync/src/app.ts +++ b/packages/sync/src/app.ts @@ -1,4 +1,8 @@ import { Logger } from "@core/logger/winston.logger"; +import { + SYNC_RECONCILE_SWEEP_EVENT, + SyncReconcileSweepEventSchema, +} from "@core/types/sync/health.contracts"; import { createInternalAuthMiddleware, createInternalServiceAuthMiddleware, @@ -37,6 +41,7 @@ import { SyncMongoService } from "@sync/storage/sync-mongo.service"; import { syncRepositories } from "@sync/storage/sync-repositories"; import { emitHealthSnapshot } from "@sync/telemetry/health-snapshot.service"; import { + captureSafely, createPostHogCaptureClient, DEFAULT_POSTHOG_HOST, type PostHogCaptureClient, @@ -225,7 +230,12 @@ async function start(): Promise { // Active, provider-configured deployments also drain jobs and renew // channels. Those register AFTER the mongo drain so teardown stops them // first and closes mongo last. - const schedulers = buildSchedulers(config, mongo); + const schedulers = buildSchedulers( + config, + mongo, + service.identity, + posthog, + ); if (schedulers) { service.shutdown.register("scheduler", async () => { // Stop every drain; each releases only its own owner's held jobs. @@ -350,6 +360,8 @@ function buildRetentionSweep(mongo: SyncMongoService): SweepScheduler { function buildSchedulers( config: SyncConfig, mongo: SyncMongoService, + identity: ReturnType, + posthog: PostHogCaptureClient | null, ): { drains: SyncScheduler[]; reconcile: SweepScheduler; @@ -425,6 +437,22 @@ function buildSchedulers( if (enqueued > 0) { logger.info(`Sync reconcile sweep enqueued ${enqueued} pull(s)`); } + // Captured on every cycle, including enqueued: 0 — this is a + // liveness heartbeat, not a backlog report (the health snapshot's + // jobs.pending already covers backlog, and is too coarse a sample + // rate to catch a queue that drains in seconds; see the event's own + // doc comment). An alert built on "N of these landed in the last + // hour" can't be fooled by sampling gaps the way a periodic gauge + // read can. + await captureSafely(posthog, { + event: SYNC_RECONCILE_SWEEP_EVENT, + distinctId: "compass-sync", + properties: SyncReconcileSweepEventSchema.parse({ + environment: identity.environment, + service: "compass-sync", + enqueued, + }), + }); return enqueued; }, },