Automate database backup verification and disaster-recovery drills

.github/workflows/verify-restore-drill.yml

@@ -0,0 +1,79 @@
+name: Verify restore drill
+
+on:
+  schedule:
+    - cron: '17 9 * * 1'
+  workflow_dispatch:
+    inputs:
+      confirmation:
+        description: Type VERIFY_RESTORE to restore into the drill target and validate it
+        required: true
+        type: string
+
+concurrency:
+  group: restore-verification-drill
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    if: github.event_name == 'schedule' || inputs.confirmation == 'VERIFY_RESTORE'
+    runs-on: ubuntu-latest
+    environment: disaster-recovery-drill
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Restore latest backup into isolated target
+        env:
+          RESTORE_COMMAND: ${{ secrets.RESTORE_DRILL_COMMAND }}
+          DRILL_DATABASE_URL: ${{ secrets.RESTORE_DRILL_DATABASE_URL }}
+          LATEST_BACKUP_ID: ${{ vars.LATEST_BACKUP_ID }}
+          LATEST_BACKUP_COMPLETED_AT: ${{ vars.LATEST_BACKUP_COMPLETED_AT }}
+        run: |
+          if [ -z "$RESTORE_COMMAND" ]; then
+            echo "RESTORE_DRILL_COMMAND must restore the latest production backup into RESTORE_DRILL_DATABASE_URL" >&2
+            exit 1
+          fi
+          eval "$RESTORE_COMMAND"
+      - name: Generate Prisma client
+        run: npm run prisma:generate --workspace=apps/api
+        env:
+          DATABASE_URL: ${{ secrets.RESTORE_DRILL_DATABASE_URL }}
+      - name: Validate restored database, encrypted wallets, and queue metadata
+        run: npm run db:verify-restore --workspace=apps/api
+        env:
+          NODE_ENV: production
+          DRILL_DATABASE_URL: ${{ secrets.RESTORE_DRILL_DATABASE_URL }}
+          REDIS_URL: ${{ secrets.RESTORE_DRILL_REDIS_URL }}
+          ENCRYPTION_KEY: ${{ secrets.RESTORE_DRILL_ENCRYPTION_KEY }}
+          LATEST_BACKUP_ID: ${{ vars.LATEST_BACKUP_ID }}
+          LATEST_BACKUP_COMPLETED_AT: ${{ vars.LATEST_BACKUP_COMPLETED_AT }}
+          BACKUP_METADATA_JSON: ${{ vars.BACKUP_METADATA_JSON }}
+          RESTORE_DRILL_MAX_BACKUP_AGE_MINUTES: ${{ vars.RESTORE_DRILL_MAX_BACKUP_AGE_MINUTES }}
+          RESTORE_DRILL_RTO_OBJECTIVE_MINUTES: ${{ vars.RESTORE_DRILL_RTO_OBJECTIVE_MINUTES }}
+          RESTORE_DRILL_QUEUE_MAX_AGE_MINUTES: ${{ vars.RESTORE_DRILL_QUEUE_MAX_AGE_MINUTES }}
+      - name: Tear down isolated restore target
+        if: always()
+        env:
+          TEARDOWN_COMMAND: ${{ secrets.RESTORE_DRILL_TEARDOWN_COMMAND }}
+          DRILL_DATABASE_URL: ${{ secrets.RESTORE_DRILL_DATABASE_URL }}
+        run: |
+          if [ -n "$TEARDOWN_COMMAND" ]; then
+            eval "$TEARDOWN_COMMAND"
+          fi
+      - name: Alert on failed drill
+        if: failure()
+        env:
+          ALERT_WEBHOOK_URL: ${{ secrets.RESTORE_DRILL_ALERT_WEBHOOK_URL }}
+        run: |
+          if [ -n "$ALERT_WEBHOOK_URL" ]; then
+            curl -fsS -X POST -H 'content-type: application/json' \
+              --data "{\"text\":\"SendAm restore verification drill failed: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}\"}" \
+              "$ALERT_WEBHOOK_URL"
+          else
+            echo "No RESTORE_DRILL_ALERT_WEBHOOK_URL configured; relying on GitHub Actions failure notifications."
+          fi

apps/api/package.json

@@ -15,7 +15,8 @@
     "db:validate": "node scripts/validate-production-db.js",
     "db:provision": "prisma migrate deploy && node scripts/validate-production-db.js",
     "whatsapp:webhook:configure": "node scripts/configure-whatsapp-webhook.js",
-    "test": "node --test"
+    "test": "node --test",
+    "db:verify-restore": "node scripts/verify-restore-drill.js"
   },
   "dependencies": {
     "@prisma/adapter-pg": "^7.8.0",

apps/api/scripts/verify-restore-drill.js

@@ -0,0 +1,153 @@
+const { Client } = require('pg');
+const { validateDatabase } = require('./validate-production-db');
+
+const DEFAULT_MAX_BACKUP_AGE_MINUTES = 24 * 60;
+const DEFAULT_RTO_OBJECTIVE_MINUTES = 60;
+const DEFAULT_QUEUE_MAX_AGE_MINUTES = 30;
+const REPRESENTATIVE_TABLES = ['User', 'Wallet', 'Transaction'];
+
+const parseTimestamp = (value, name) => {
+  if (!value) throw new Error(`${name} is required`);
+  const date = new Date(value);
+  if (Number.isNaN(date.getTime())) throw new Error(`${name} must be an ISO-8601 timestamp`);
+  return date;
+};
+
+const minutesBetween = (later, earlier) => Math.max(0, Math.round((later.getTime() - earlier.getTime()) / 60000));
+
+const parsePositiveInteger = (value, fallback, name) => {
+  if (value === undefined || value === '') return fallback;
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`);
+  return parsed;
+};
+
+const readBackupMetadata = (env) => {
+  if (env.BACKUP_METADATA_JSON) {
+    const metadata = JSON.parse(env.BACKUP_METADATA_JSON);
+    return {
+      completedAt: metadata.completedAt || metadata.completed_at,
+      backupId: metadata.backupId || metadata.backup_id || 'metadata-json',
+    };
+  }
+  return {
+    completedAt: env.LATEST_BACKUP_COMPLETED_AT,
+    backupId: env.LATEST_BACKUP_ID || 'latest',
+  };
+};
+
+const validateBackupFreshness = ({ completedAt, now, maxAgeMinutes }) => {
+  const backupCompletedAt = parseTimestamp(completedAt, 'latest backup completion time');
+  const rpoMinutes = minutesBetween(now, backupCompletedAt);
+  if (backupCompletedAt.getTime() > now.getTime()) throw new Error('latest backup completion time cannot be in the future');
+  if (rpoMinutes > maxAgeMinutes) {
+    throw new Error(`Latest backup is stale: ${rpoMinutes} minutes old exceeds ${maxAgeMinutes} minute RPO`);
+  }
+  return { backupCompletedAt: backupCompletedAt.toISOString(), rpoMinutes };
+};
+
+const validateRedisQueue = async ({ redisUrl, maxAgeMinutes, redisFactory }) => {
+  if (!redisUrl) return { checked: false, reason: 'REDIS_URL was not provided' };
+  const Redis = redisFactory || require('ioredis');
+  const redis = new Redis(redisUrl, { maxRetriesPerRequest: 1, enableReadyCheck: true });
+  try {
+    const queueNames = ['whatsapp'];
+    const waitingCounts = await Promise.all(queueNames.map((name) => redis.llen(`bull:${name}:wait`)));
+    const delayedCounts = await Promise.all(queueNames.map((name) => redis.zcard(`bull:${name}:delayed`)));
+    const failedCounts = await Promise.all(queueNames.map((name) => redis.zcard(`bull:${name}:failed`)));
+    return {
+      checked: true,
+      maxQueueAgeMinutes: maxAgeMinutes,
+      waitingJobs: waitingCounts.reduce((sum, count) => sum + Number(count || 0), 0),
+      delayedJobs: delayedCounts.reduce((sum, count) => sum + Number(count || 0), 0),
+      failedJobs: failedCounts.reduce((sum, count) => sum + Number(count || 0), 0),
+    };
+  } finally {
+    if (typeof redis.quit === 'function') await redis.quit().catch(() => redis.disconnect?.());
+    else redis.disconnect?.();
+  }
+};
+
+const queryRepresentativeCounts = async (client) => {
+  const counts = {};
+  for (const table of REPRESENTATIVE_TABLES) {
+    const result = await client.query(`SELECT count(*)::int AS count FROM "${table}"`);
+    counts[table] = result.rows[0]?.count || 0;
+  }
+  return counts;
+};
+
+const verifyWalletDecryptability = async (client, decrypt) => {
+  const result = await client.query('SELECT id, "encryptedSecretKey" FROM "Wallet" WHERE "encryptedSecretKey" IS NOT NULL LIMIT 5');
+  let checked = 0;
+  for (const row of result.rows) {
+    decrypt(row.encryptedSecretKey);
+    checked += 1;
+  }
+  return { checked, sampleSize: result.rows.length };
+};
+
+const runRestoreDrill = async ({
+  env = process.env,
+  now = new Date(),
+  clientFactory = (config) => new Client(config),
+  redisFactory,
+  decrypt,
+} = {}) => {
+  const startedAt = now;
+  const connectionString = env.DATABASE_URL || env.DRILL_DATABASE_URL;
+  if (!connectionString) throw new Error('DRILL_DATABASE_URL or DATABASE_URL is required');
+
+  const maxAgeMinutes = parsePositiveInteger(env.RESTORE_DRILL_MAX_BACKUP_AGE_MINUTES, DEFAULT_MAX_BACKUP_AGE_MINUTES, 'RESTORE_DRILL_MAX_BACKUP_AGE_MINUTES');
+  const rtoObjectiveMinutes = parsePositiveInteger(env.RESTORE_DRILL_RTO_OBJECTIVE_MINUTES, DEFAULT_RTO_OBJECTIVE_MINUTES, 'RESTORE_DRILL_RTO_OBJECTIVE_MINUTES');
+  const queueMaxAgeMinutes = parsePositiveInteger(env.RESTORE_DRILL_QUEUE_MAX_AGE_MINUTES, DEFAULT_QUEUE_MAX_AGE_MINUTES, 'RESTORE_DRILL_QUEUE_MAX_AGE_MINUTES');
+  const metadata = readBackupMetadata(env);
+  const freshness = validateBackupFreshness({ completedAt: metadata.completedAt, now: startedAt, maxAgeMinutes });
+
+  const schema = await validateDatabase({ connectionString, nodeEnv: env.NODE_ENV || 'production', clientFactory });
+  const client = clientFactory({ connectionString, connectionTimeoutMillis: 10000 });
+  try {
+    await client.connect();
+    const counts = await queryRepresentativeCounts(client);
+    const walletDecrypt = await verifyWalletDecryptability(client, decrypt || require('../src/services/crypto.service').decrypt);
+    const queue = await validateRedisQueue({ redisUrl: env.REDIS_URL || env.UPSTASH_REDIS_URL, maxAgeMinutes: queueMaxAgeMinutes, redisFactory });
+    const finishedAt = new Date();
+    const rtoMinutes = Math.max(1, minutesBetween(finishedAt, startedAt));
+    if (rtoMinutes > rtoObjectiveMinutes) throw new Error(`Restore drill RTO ${rtoMinutes} minutes exceeds ${rtoObjectiveMinutes} minute objective`);
+    return {
+      event: 'restore_drill_passed',
+      backupId: metadata.backupId,
+      backupCompletedAt: freshness.backupCompletedAt,
+      rpoMinutes: freshness.rpoMinutes,
+      rtoMinutes,
+      rtoObjectiveMinutes,
+      schema,
+      representativeCounts: counts,
+      walletDecrypt,
+      queue,
+      evidenceRedacted: true,
+    };
+  } finally {
+    await client.end().catch(() => {});
+  }
+};
+
+const run = async () => {
+  try {
+    const result = await runRestoreDrill();
+    console.log(JSON.stringify(result));
+  } catch (error) {
+    console.error(JSON.stringify({ event: 'restore_drill_failed', error: error.message }));
+    process.exitCode = 1;
+  }
+};
+
+if (require.main === module) run();
+
+module.exports = {
+  runRestoreDrill,
+  validateBackupFreshness,
+  readBackupMetadata,
+  validateRedisQueue,
+  DEFAULT_MAX_BACKUP_AGE_MINUTES,
+};

apps/api/test/restore-drill.verification.test.js

@@ -0,0 +1,105 @@
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const fs = require('node:fs');
+const { REQUIRED_TABLES } = require('../scripts/validate-production-db');
+const {
+  runRestoreDrill,
+  validateBackupFreshness,
+  readBackupMetadata,
+  validateRedisQueue,
+} = require('../scripts/verify-restore-drill');
+
+const migrationDirectory = path.resolve(__dirname, '../prisma/migrations');
+
+const successfulClient = () => ({
+  ended: false,
+  async connect() {},
+  async end() { this.ended = true; },
+  async query(sql) {
+    if (sql === 'SHOW server_version') return { rows: [{ server_version: '16.4' }] };
+    if (sql.includes('to_regclass')) return { rows: [{ name: '_prisma_migrations' }] };
+    if (sql.includes('finished_at IS NULL')) return { rows: [] };
+    if (sql.includes('SELECT migration_name')) {
+      return {
+        rows: fs.readdirSync(migrationDirectory, { withFileTypes: true })
+          .filter((entry) => entry.isDirectory())
+          .map((entry) => ({ migration_name: entry.name })),
+      };
+    }
+    if (sql.includes('pg_catalog.pg_tables')) {
+      return { rows: REQUIRED_TABLES.map((tablename) => ({ tablename })) };
+    }
+    if (sql.includes('count(*)::int')) return { rows: [{ count: 7 }] };
+    if (sql.includes('encryptedSecretKey')) {
+      return { rows: [{ id: 'wallet_1', encryptedSecretKey: 'ciphertext' }] };
+    }
+    throw new Error(`Unexpected query: ${sql}`);
+  },
+});
+
+test('reads backup metadata from JSON without exposing provider details', () => {
+  assert.deepEqual(
+    readBackupMetadata({ BACKUP_METADATA_JSON: '{"backupId":"backup-123","completedAt":"2026-08-20T09:00:00.000Z"}' }),
+    { backupId: 'backup-123', completedAt: '2026-08-20T09:00:00.000Z' },
+  );
+});
+
+test('fails stale backups against the configured RPO threshold', () => {
+  assert.throws(
+    () => validateBackupFreshness({
+      completedAt: '2026-08-19T00:00:00.000Z',
+      now: new Date('2026-08-20T09:00:00.000Z'),
+      maxAgeMinutes: 60,
+    }),
+    /Latest backup is stale/,
+  );
+});
+
+test('summarizes restored Redis queue state without payloads', async () => {
+  const calls = [];
+  class FakeRedis {
+    constructor(url) { this.url = url; }
+    async llen(key) { calls.push(key); return 2; }
+    async zcard(key) { calls.push(key); return 3; }
+    async quit() {}
+  }
+
+  const result = await validateRedisQueue({
+    redisUrl: 'redis://localhost:6379',
+    maxAgeMinutes: 30,
+    redisFactory: FakeRedis,
+  });
+
+  assert.equal(result.checked, true);
+  assert.equal(result.waitingJobs, 2);
+  assert.equal(result.delayedJobs, 3);
+  assert.equal(result.failedJobs, 3);
+  assert.deepEqual(calls, ['bull:whatsapp:wait', 'bull:whatsapp:delayed', 'bull:whatsapp:failed']);
+});
+
+test('runs schema, representative data, wallet decryptability, RPO, and RTO checks', async () => {
+  const clients = [successfulClient(), successfulClient()];
+  const result = await runRestoreDrill({
+    env: {
+      DRILL_DATABASE_URL: 'postgresql://sendam:secret@localhost/drill',
+      LATEST_BACKUP_ID: 'backup-123',
+      LATEST_BACKUP_COMPLETED_AT: '2026-08-20T08:30:00.000Z',
+      RESTORE_DRILL_MAX_BACKUP_AGE_MINUTES: '120',
+      RESTORE_DRILL_RTO_OBJECTIVE_MINUTES: '60',
+    },
+    now: new Date('2026-08-20T09:00:00.000Z'),
+    clientFactory: () => clients.shift(),
+    decrypt: (value) => {
+      assert.equal(value, 'ciphertext');
+      return 'plaintext-secret';
+    },
+  });
+
+  assert.equal(result.event, 'restore_drill_passed');
+  assert.equal(result.backupId, 'backup-123');
+  assert.equal(result.rpoMinutes, 30);
+  assert.equal(result.representativeCounts.User, 7);
+  assert.equal(result.walletDecrypt.checked, 1);
+  assert.equal(result.evidenceRedacted, true);
+});

docs/BACKGROUND-WORKERS.md

@@ -99,3 +99,30 @@ deploy the previous API image. Never run both a legacy poller and the new poller
 for longer than the controlled overlap. Redis jobs are forward-compatible
 because queue and job names are unchanged; do not delete Redis or failed jobs
 during rollback.
+
+## Queue backup and restore drills
+
+Redis-backed BullMQ state is part of the disaster-recovery plan because accepted
+webhooks may be waiting, delayed, failed, or stalled when PostgreSQL is restored.
+Platform Engineering owns Redis backup/PITR settings and validates them during
+the **Verify restore drill** workflow. Backend owners own safe replay guidance,
+and Compliance plus Payments must approve replay for jobs that may have crossed
+an external financial boundary.
+
+Queue recovery objectives are:
+
+- **Redis queue-state RPO:** latest durable Redis snapshot or provider restore
+  point must be no older than 30 minutes for BullMQ wait/delayed/failed sets.
+- **Queue recovery RTO:** Redis restore and worker drain validation must fit
+  inside the 60-minute application RTO.
+
+During a drill, restore Redis into an isolated target and set
+`RESTORE_DRILL_REDIS_URL`. The verifier summarizes BullMQ wait, delayed, and
+failed counts without logging job payloads. If Redis cannot be restored, record
+that dependency gap in the evidence and keep the PostgreSQL drill failed until a
+queue recovery plan is proven.
+
+After an incident restore, start workers only after PostgreSQL validation and
+key access checks pass. Inspect failed or waiting jobs by redacted job ID, avoid
+copying payloads into tickets, reconcile any payment-related transaction before
+replay, and document the final queue depth in the incident evidence.