feat(selfhost): add a live dead-letter-rate gauge sampled from the audit ledger (#3244)

Author: joaovictor91123

src/selfhost/dlq-recent.ts

@@ -0,0 +1,22 @@
+import { countRecentDeadLetters } from "../db/repositories";
+
+// Trailing window for the "is the DLQ dead-lettering right now?" gauge (#2083). Operators alert on the RATE of
+// recent DLQ-consumer drops, which the cumulative `gittensory_dlq_dead_lettered_total` counter and the point-in-time
+// `gittensory_queue_dead` depth gauge can't express on their own.
+export const DLQ_RECENT_WINDOW_MS = 15 * 60 * 1000; // 15 minutes
+
+/** ISO-8601 timestamp `windowMs` before `now` (default: current time). Pure given `now`; the injectable clock keeps
+ *  the window math deterministic in tests, and matches the ISO-compare convention used by the queue reliability work. */
+export function isoNowMinus(windowMs: number, now: number = Date.now()): string {
+  return new Date(now - windowMs).toISOString();
+}
+
+/** Scrape-time sample of DLQ dead-letters within the trailing window. Swallows a query error so a transient DB
+ *  hiccup degrades the sample to 0 rather than rejecting and breaking the whole `/metrics` scrape. */
+export async function sampleRecentDeadLetters(env: Env, now: number = Date.now()): Promise<number> {
+  try {
+    return await countRecentDeadLetters(env, isoNowMinus(DLQ_RECENT_WINDOW_MS, now));
+  } catch {
+    return 0;
+  }
+}

src/selfhost/metrics.ts

@@ -36,6 +36,7 @@ const histograms = new Map<string, HistogramState>();
 const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["gittensory_queue_pending", { help: "Current in-process queue depth.", type: "gauge" }],
   ["gittensory_queue_dead", { help: "Current in-process dead queue depth.", type: "gauge" }],
+  ["gittensory_dlq_dead_lettered_recent", { help: "DLQ messages dead-lettered within the recent trailing window, sampled at scrape.", type: "gauge" }],
   ["gittensory_queue_processing", { help: "Jobs currently claimed and mid-flight.", type: "gauge" }],
   ["gittensory_queue_runnable_now", { help: "Pending jobs, any priority, currently due (run_after<=now).", type: "gauge" }],
   ["gittensory_queue_live_pending", { help: "Current live-work queue depth.", type: "gauge" }],

src/server.ts

@@ -98,6 +98,7 @@ import {
   setLocalReviewContextReader,
 } from "./signals/focus-manifest-loader";
 import { probeReesSecretAtStartup } from "./review/enrichment-wire";
+import { sampleRecentDeadLetters } from "./selfhost/dlq-recent";
 import type { JobMessage } from "./types";
 
 /** Resolve `<NAME>_FILE` env vars (Docker secrets / multi-line keys) into `<NAME>` at startup. */
@@ -621,6 +622,7 @@ async function main(): Promise<void> {
 
   gauge("gittensory_queue_pending", () => backend.queue.size());
   gauge("gittensory_queue_dead", () => backend.queue.deadCount());
+  gauge("gittensory_dlq_dead_lettered_recent", () => sampleRecentDeadLetters(env));
   gauge("gittensory_queue_processing", () => backend.queue.processingCount());
   const durableJobMetric = async (name: string): Promise<number> =>
     Number((await backend.queue.stats())[name] ?? 0);

test/unit/dlq-recent.test.ts

@@ -0,0 +1,37 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import * as repositories from "../../src/db/repositories";
+import { DLQ_RECENT_WINDOW_MS, isoNowMinus, sampleRecentDeadLetters } from "../../src/selfhost/dlq-recent";
+
+describe("dlq-recent gauge helpers (#2083)", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  describe("isoNowMinus", () => {
+    it("returns the ISO timestamp windowMs before the injected now", () => {
+      const now = Date.parse("2026-07-04T12:00:00.000Z");
+      expect(isoNowMinus(15 * 60 * 1000, now)).toBe("2026-07-04T11:45:00.000Z");
+    });
+
+    it("defaults to the current clock when no now is given", () => {
+      // Default-parameter path: a window before "now" is always in the past.
+      expect(isoNowMinus(1000) <= new Date().toISOString()).toBe(true);
+    });
+
+    it("exposes a 15-minute default window", () => {
+      expect(DLQ_RECENT_WINDOW_MS).toBe(900_000);
+    });
+  });
+
+  describe("sampleRecentDeadLetters", () => {
+    it("returns the count over the trailing window, queried at the window start", async () => {
+      const now = Date.parse("2026-07-04T12:00:00.000Z");
+      const spy = vi.spyOn(repositories, "countRecentDeadLetters").mockResolvedValue(7);
+      expect(await sampleRecentDeadLetters({} as Env, now)).toBe(7);
+      expect(spy).toHaveBeenCalledWith({}, "2026-07-04T11:45:00.000Z");
+    });
+
+    it("degrades to 0 when the query throws, so a DB hiccup never breaks the scrape", async () => {
+      vi.spyOn(repositories, "countRecentDeadLetters").mockRejectedValue(new Error("db down"));
+      expect(await sampleRecentDeadLetters({} as Env)).toBe(0);
+    });
+  });
+});