fix(awareness-service): stop re-claiming exhausted deliveries

A dead-lettered delivery was left with status 'failed', which claimBatch
selects - so the engine re-claimed it every tick, re-attempted it, inflated
its attempt count without bound and inserted a fresh dead-letter row each
time. Mark exhausted deliveries with a terminal 'dead' status and exclude any
delivery at/over the attempt cap from claiming.

Author: coodos

services/awareness-service/api/src/database/entities/Delivery.ts

@@ -11,7 +11,8 @@ export type DeliveryStatus =
     | "pending"
     | "delivering"
     | "delivered"
-    | "failed";
+    | "failed"
+    | "dead";
 
 /**
  * A queued webhook delivery of one packet to one subscription. The unique

services/awareness-service/api/src/openapi.ts

@@ -107,7 +107,13 @@ export const openApiDocument = {
                     packetId: { type: "string" },
                     status: {
                         type: "string",
-                        enum: ["pending", "delivering", "delivered", "failed"],
+                        enum: [
+                            "pending",
+                            "delivering",
+                            "delivered",
+                            "failed",
+                            "dead",
+                        ],
                     },
                     attempts: { type: "integer" },
                     nextAttemptAt: { type: "string", format: "date-time" },

services/awareness-service/api/src/services/DeliveryEngine.ts

@@ -92,15 +92,19 @@ export class DeliveryEngine {
             .update(Delivery)
             .set({ status: "delivering" })
             .where(
+                // Only pending/failed deliveries still under the attempt
+                // limit are claimable. `dead` deliveries (and any that
+                // already hit the cap) are terminal and never re-claimed.
                 `id IN (
                     SELECT id FROM deliveries
                     WHERE status IN ('pending', 'failed')
+                      AND attempts < :maxAttempts
                       AND "nextAttemptAt" <= now()
                     ORDER BY "nextAttemptAt"
                     LIMIT :limit
                     FOR UPDATE SKIP LOCKED
                 )`,
-                { limit: BATCH_SIZE },
+                { limit: BATCH_SIZE, maxAttempts: config.maxAttempts },
             )
             .returning("*")
             .execute();
@@ -181,8 +185,11 @@ export class DeliveryEngine {
         const deliveryRepo = AppDataSource.getRepository(Delivery);
 
         if (attempts >= config.maxAttempts) {
+            // Terminal: mark `dead` so the engine never re-claims it. (Using
+            // `failed` here let exhausted deliveries be picked up again every
+            // tick, inflating attempts and spawning duplicate dead letters.)
             await deliveryRepo.update(delivery.id, {
-                status: "failed",
+                status: "dead",
                 attempts,
                 lastError: message,
                 lastResponseStatus: responseStatus,