Skip to content

Commit ade45d5

Browse files
refactor(sync,scripts): harden exhausted-job cli and share filter
Use a true exhausted count with truncation signal on list, reject clear on non-failed jobs, and check remove success. Share the exhausted-job Mongo filter between count/list helpers. Co-authored-by: Tyler Dane <tyler-dane@users.noreply.github.com>
1 parent 01b4b48 commit ade45d5

3 files changed

Lines changed: 37 additions & 35 deletions

File tree

packages/scripts/src/commands/manage-failed-jobs.ts

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,6 @@ function flagValue(args: string[], name: string): string | undefined {
2525
return args[index + 1];
2626
}
2727

28-
function hasFlag(args: string[], name: string): boolean {
29-
return args.includes(name);
30-
}
31-
3228
/**
3329
* Operator tooling for Sync jobs that exhausted the self-heal requeue budget
3430
* and still occupy a coalescing key. Default actions are dry-run; `--apply`
@@ -47,7 +43,7 @@ export async function runManageFailedJobs(): Promise<void> {
4743
);
4844
}
4945

50-
const apply = hasFlag(args, "--apply");
46+
const apply = args.includes("--apply");
5147
const syncMongo = new SyncMongoService();
5248
try {
5349
await syncMongo.connect({
@@ -58,11 +54,16 @@ export async function runManageFailedJobs(): Promise<void> {
5854
const jobs = new JobRepository(syncMongo.db);
5955

6056
if (action === "list") {
61-
const exhausted = await jobs.listExhaustedFailed(FAILED_JOB_MAX_REQUEUES);
57+
const [count, sample] = await Promise.all([
58+
jobs.countExhaustedFailed(FAILED_JOB_MAX_REQUEUES),
59+
jobs.listExhaustedFailed(FAILED_JOB_MAX_REQUEUES),
60+
]);
6261
const report = {
6362
maxRequeues: FAILED_JOB_MAX_REQUEUES,
64-
count: exhausted.length,
65-
jobs: exhausted.map((job) => ({
63+
count,
64+
sampleSize: sample.length,
65+
truncated: count > sample.length,
66+
jobs: sample.map((job) => ({
6667
id: job.id,
6768
coalescingKey: job.coalescingKey,
6869
connectionId: job.connectionId,
@@ -73,7 +74,7 @@ export async function runManageFailedJobs(): Promise<void> {
7374
};
7475
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
7576
logger.info(
76-
`manage-failed-jobs list count=${report.count} maxRequeues=${FAILED_JOB_MAX_REQUEUES}`,
77+
`manage-failed-jobs list count=${report.count} sample=${report.sampleSize} truncated=${report.truncated}`,
7778
);
7879
await syncMongo.disconnect();
7980
process.exit(0);
@@ -88,6 +89,11 @@ export async function runManageFailedJobs(): Promise<void> {
8889
if (!existing) {
8990
throw new Error(`No job found for id=${id}`);
9091
}
92+
if (existing.state !== "failed") {
93+
throw new Error(
94+
`Job id=${id} is state=${existing.state}; ${action} only accepts failed jobs`,
95+
);
96+
}
9197

9298
if (action === "clear") {
9399
const coalescingKey =
@@ -99,13 +105,16 @@ export async function runManageFailedJobs(): Promise<void> {
99105
}
100106
const report = {
101107
dryRun: !apply,
102-
action: "clear",
108+
action: "clear" as const,
103109
id,
104110
coalescingKey,
105111
state: existing.state,
106112
};
107113
if (apply) {
108-
await jobs.remove(id, coalescingKey);
114+
const ok = await jobs.remove(id, coalescingKey);
115+
if (!ok) {
116+
throw new Error(`Failed to clear id=${id} key=${coalescingKey}`);
117+
}
109118
}
110119
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
111120
logger.info(
@@ -115,15 +124,9 @@ export async function runManageFailedJobs(): Promise<void> {
115124
process.exit(0);
116125
}
117126

118-
// requeue
119-
if (existing.state !== "failed") {
120-
throw new Error(
121-
`Job id=${id} is state=${existing.state}; requeue only accepts failed jobs`,
122-
);
123-
}
124127
const report = {
125128
dryRun: !apply,
126-
action: "requeue",
129+
action: "requeue" as const,
127130
id,
128131
coalescingKey: existing.coalescingKey,
129132
requeuedCount: existing.requeuedCount,

packages/sync/src/storage/repositories/job.repository.ts

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ export type ExhaustedFailedJob = {
2222
updatedAt: Date;
2323
};
2424

25+
function exhaustedFailedFilter(maxRequeues: number) {
26+
return {
27+
state: "failed" as const,
28+
failureClass: { $ne: "permanent" as const },
29+
requeuedCount: { $gte: maxRequeues },
30+
};
31+
}
32+
2533
// Repository for `jobs`. Enqueue coalesces on a unique key so repeated
2634
// notifications for the same resource collapse into one pending job instead of
2735
// an unbounded queue. Terminal jobs are removed so a later notification can
@@ -283,11 +291,7 @@ export class JobRepository {
283291
// the sweep will not touch them again; an operator must. Used to drive a
284292
// loud, recurring alert rather than a silent terminal state.
285293
async countExhaustedFailed(maxRequeues: number): Promise<number> {
286-
return this.collection.countDocuments({
287-
state: "failed",
288-
failureClass: { $ne: "permanent" },
289-
requeuedCount: { $gte: maxRequeues },
290-
});
294+
return this.collection.countDocuments(exhaustedFailedFilter(maxRequeues));
291295
}
292296

293297
// Same filter as countExhaustedFailed, returning the rows an operator needs
@@ -297,11 +301,7 @@ export class JobRepository {
297301
limit = 50,
298302
): Promise<ExhaustedFailedJob[]> {
299303
const rows = await this.collection
300-
.find({
301-
state: "failed",
302-
failureClass: { $ne: "permanent" },
303-
requeuedCount: { $gte: maxRequeues },
304-
})
304+
.find(exhaustedFailedFilter(maxRequeues))
305305
.sort({ updatedAt: 1 })
306306
.limit(limit)
307307
.project({
@@ -315,7 +315,7 @@ export class JobRepository {
315315
.toArray();
316316

317317
return rows.map((row) => ({
318-
id: row._id as SyncJobId,
318+
id: row["_id"] as SyncJobId,
319319
coalescingKey: String(row["coalescingKey"]),
320320
connectionId: row["connectionId"] as ConnectionId,
321321
failureClass: (row["failureClass"] ?? "retryableTransient") as Exclude<
@@ -351,9 +351,7 @@ export class JobRepository {
351351
tenantId,
352352
principalId,
353353
connectionId,
354-
state: "failed",
355-
failureClass: { $ne: "permanent" },
356-
requeuedCount: { $gte: maxRequeues },
354+
...exhaustedFailedFilter(maxRequeues),
357355
});
358356
}
359357

packages/web/src/events/grid-event-draft.adapter.test.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { Origin } from "@core/constants/core.constants";
12
import {
23
type Calendar,
34
getCalendarCapabilities,
@@ -150,17 +151,17 @@ test("projects a grid draft into a grid event without a CompassEvent bridge", ()
150151
expect(allDayGrid.isAllDay).toBe(true);
151152
expect(allDayGrid.title).toBe("All day");
152153
expect(allDayGrid.calendarId).toBe(timedEvent.calendarId);
153-
expect(allDayGrid.origin).toBe("compass");
154+
expect(allDayGrid.origin).toBe(Origin.COMPASS);
154155

155156
const timed = editGridEventDraft(timedEvent);
156157
if (!timed) throw new Error("Expected timed edit draft");
157-
timed.values.color = "tomato";
158+
timed.values.color = "coral";
158159
const timedGrid = gridEventDraftToGridEvent(timed);
159160

160161
expect(timedGrid.isAllDay).toBe(false);
161162
expect(timedGrid.startDate).toBe(dayjs(timed.values.schedule.start).format());
162163
expect(timedGrid.endDate).toBe(dayjs(timed.values.schedule.end).format());
163-
expect(timedGrid.color).toBe("tomato");
164+
expect(timedGrid.color).toBe("coral");
164165
expect(timedGrid.isBusy).toBe(false);
165166
});
166167

0 commit comments

Comments
 (0)