diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml new file mode 100644 index 0000000..b8daf56 --- /dev/null +++ b/.github/workflows/backup.yml @@ -0,0 +1,73 @@ +name: Database Backup + +# Scheduled offsite backup of the production database. +# +# Runs daily at 03:00 UTC. The job creates a verified backup and uploads it to +# S3 (a second, offsite location for disaster recovery). The backup artifact is +# also attached to the workflow run as a third copy / audit trail. +# +# Required repository secrets (Settings → Secrets and variables → Actions): +# DATABASE_URL PostgreSQL connection string of the production DB +# BACKUP_S3_BUCKET target S3 bucket +# AWS_ACCESS_KEY_ID credentials with s3:PutObject on the bucket +# AWS_SECRET_ACCESS_KEY +# Optional: +# BACKUP_S3_PREFIX (default: backups), BACKUP_S3_REGION (default: us-east-1) + +on: + schedule: + - cron: '0 3 * * *' # daily at 03:00 UTC + workflow_dispatch: {} # allow manual runs from the Actions tab + +concurrency: + group: db-backup + cancel-in-progress: false + +jobs: + backup: + name: Create & upload backup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: backend/package-lock.json + + - name: Install dependencies + working-directory: backend + run: npm ci + + - name: Install S3 client + working-directory: backend + run: npm install --no-save @aws-sdk/client-s3 + + - name: Ensure PostgreSQL client is available + run: | + if ! command -v pg_dump >/dev/null; then + sudo apt-get update && sudo apt-get install -y postgresql-client + fi + pg_dump --version + + - name: Run backup + working-directory: backend + env: + NODE_ENV: production + DATABASE_URL: ${{ secrets.DATABASE_URL }} + BACKUP_S3_BUCKET: ${{ secrets.BACKUP_S3_BUCKET }} + BACKUP_S3_PREFIX: ${{ secrets.BACKUP_S3_PREFIX }} + BACKUP_S3_REGION: ${{ secrets.BACKUP_S3_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: npm run backup + + - name: Upload backup artifact (retained 30 days) + if: always() + uses: actions/upload-artifact@v4 + with: + name: db-backup-${{ github.run_id }} + path: backend/backups/ + retention-days: 30 + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index b891e64..d11c666 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,12 @@ webhooks.json test-data.json .blue-green-state.json +# Database files and backups +*.db +backups/ +restore-database.sql +*.pre-restore + # Playwright test-results/ playwright-report/ diff --git a/README.md b/README.md index c6ecb89..3b38188 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ graph TB - [Troubleshooting Guide](docs/troubleshooting.md) — Common issues and solutions - [Multi-Region Deployment](docs/multi-region-deployment.md) — Deployment strategy and failover - [Kubernetes Deployment](docs/kubernetes-deployment.md) — Kubernetes manifests, scaling, and self-healing +- [Database Backup & Restore](docs/backups.md) — Automated backups, S3 offsite storage, retention, and disaster recovery - [NFT Certificates](docs/NFT_CERTIFICATES.md) - [NFT Quickstart](docs/NFT_QUICKSTART.md) - [FAQ](docs/FAQ.md) diff --git a/backend/__tests__/backup.test.js b/backend/__tests__/backup.test.js new file mode 100644 index 0000000..72d5820 --- /dev/null +++ b/backend/__tests__/backup.test.js @@ -0,0 +1,171 @@ +process.env.NODE_ENV = 'test'; + +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { gunzipSync } from 'zlib'; +import { + loadBackupConfig, + createBackup, + verifyBackup, + listBackups, + selectExpired, + rotateBackups, + parseBackupTimestamp, +} from '../backup.js'; +import { restoreBackup } from '../restore.js'; + +let root; + +function makeConfig(overrides = {}) { + return { + backendDir: root, + backupDir: join(root, 'backups'), + dataFiles: [join(root, 'data.json'), join(root, 'webhooks.json')], + databaseUrl: './does-not-exist.db', // no SQLite file → db source skipped + retentionDays: 7, + retentionMax: 30, + s3: null, + ...overrides, + }; +} + +function entry(name) { + return { name, dir: `/x/${name}`, createdAt: parseBackupTimestamp(name) }; +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'backup-test-')); + writeFileSync(join(root, 'data.json'), JSON.stringify({ asset: 1 })); + writeFileSync(join(root, 'webhooks.json'), JSON.stringify({ hook: 2 })); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// ── Config ──────────────────────────────────────────────────────────────────── +describe('loadBackupConfig', () => { + test('applies sane defaults', () => { + const cfg = loadBackupConfig({}); + expect(cfg.retentionDays).toBe(7); + expect(cfg.retentionMax).toBe(30); + expect(cfg.s3).toBeNull(); + expect(cfg.dataFiles.some((f) => f.endsWith('data.json'))).toBe(true); + expect(cfg.dataFiles.some((f) => f.endsWith('webhooks.json'))).toBe(true); + }); + + test('parses S3 config when bucket is set', () => { + const cfg = loadBackupConfig({ BACKUP_S3_BUCKET: 'b', BACKUP_S3_PREFIX: 'p/', BACKUP_S3_REGION: 'eu-west-1' }); + expect(cfg.s3).toEqual({ bucket: 'b', prefix: 'p', region: 'eu-west-1', endpoint: undefined }); + }); +}); + +// ── Create + verify ───────────────────────────────────────────────────────────── +describe('createBackup / verifyBackup', () => { + test('creates gzipped members + manifest and verifies clean', async () => { + const cfg = makeConfig(); + const { dir, manifest } = await createBackup(cfg, { now: new Date('2026-06-30T03:00:00.000Z') }); + + expect(existsSync(join(dir, 'data.json.gz'))).toBe(true); + expect(existsSync(join(dir, 'webhooks.json.gz'))).toBe(true); + expect(existsSync(join(dir, 'manifest.json'))).toBe(true); + expect(manifest.members).toHaveLength(2); + + const v = await verifyBackup(dir); + expect(v.ok).toBe(true); + expect(v.results.every((r) => r.ok)).toBe(true); + + // gzip member round-trips to original bytes + const restored = JSON.parse(gunzipSync(readFileSync(join(dir, 'data.json.gz')))); + expect(restored).toEqual({ asset: 1 }); + }); + + test('verification fails when a member is tampered with', async () => { + const cfg = makeConfig(); + const { dir } = await createBackup(cfg, { now: new Date('2026-06-30T03:00:00.000Z') }); + + writeFileSync(join(dir, 'data.json.gz'), Buffer.from('not gzip')); + const v = await verifyBackup(dir); + expect(v.ok).toBe(false); + expect(v.results.find((r) => r.name === 'data.json.gz').ok).toBe(false); + }); + + test('throws when no data source exists', async () => { + const cfg = makeConfig({ dataFiles: [join(root, 'nope.json')] }); + await expect(createBackup(cfg, {})).rejects.toThrow(/No data sources/); + }); +}); + +// ── Retention selection (pure) ─────────────────────────────────────────────────── +describe('selectExpired', () => { + test('expires backups older than retentionDays', () => { + const backups = [ + entry('backup-2026-06-01T03-00-00-000Z'), // ~29 days old + entry('backup-2026-06-29T03-00-00-000Z'), // 1 day old + ]; + const now = new Date('2026-06-30T03:00:00.000Z'); + const expired = selectExpired(backups, { retentionDays: 7, retentionMax: 0 }, now); + expect(expired.map((b) => b.name)).toEqual(['backup-2026-06-01T03-00-00-000Z']); + }); + + test('caps total count, dropping oldest first', () => { + const backups = [ + entry('backup-2026-06-28T03-00-00-000Z'), + entry('backup-2026-06-29T03-00-00-000Z'), + entry('backup-2026-06-30T03-00-00-000Z'), + ]; + const now = new Date('2026-06-30T04:00:00.000Z'); + const expired = selectExpired(backups, { retentionDays: 0, retentionMax: 2 }, now); + expect(expired.map((b) => b.name)).toEqual(['backup-2026-06-28T03-00-00-000Z']); + }); + + test('retains everything when both limits are 0', () => { + const backups = [entry('backup-2020-01-01T00-00-00-000Z')]; + const expired = selectExpired(backups, { retentionDays: 0, retentionMax: 0 }, new Date()); + expect(expired).toHaveLength(0); + }); +}); + +// ── Rotation (I/O) ────────────────────────────────────────────────────────────── +describe('rotateBackups', () => { + test('deletes expired backup directories', async () => { + const cfg = makeConfig({ retentionDays: 0, retentionMax: 1 }); + mkdirSync(cfg.backupDir, { recursive: true }); + for (const name of ['backup-2026-06-28T03-00-00-000Z', 'backup-2026-06-30T03-00-00-000Z']) { + const d = join(cfg.backupDir, name); + mkdirSync(d, { recursive: true }); + writeFileSync(join(d, 'manifest.json'), '{}'); + } + + const removed = rotateBackups(cfg, new Date('2026-06-30T04:00:00.000Z')); + expect(removed).toEqual(['backup-2026-06-28T03-00-00-000Z']); + expect(listBackups(cfg).map((b) => b.name)).toEqual(['backup-2026-06-30T03-00-00-000Z']); + }); +}); + +// ── Restore ───────────────────────────────────────────────────────────────────── +describe('restoreBackup', () => { + test('restores data files, preserving the current copy', async () => { + const cfg = makeConfig(); + const { name } = await createBackup(cfg, { now: new Date('2026-06-30T03:00:00.000Z') }); + + // Mutate live data, then restore + writeFileSync(join(root, 'data.json'), JSON.stringify({ asset: 999 })); + const written = await restoreBackup(name, { config: cfg, log: { info() {}, warn() {}, error() {} } }); + + expect(JSON.parse(readFileSync(join(root, 'data.json'), 'utf-8'))).toEqual({ asset: 1 }); + expect(existsSync(join(root, 'data.json.pre-restore'))).toBe(true); + expect(written.some((p) => p.endsWith('data.json'))).toBe(true); + }); + + test('refuses a corrupt backup unless forced', async () => { + const cfg = makeConfig(); + const { name, dir } = await createBackup(cfg, { now: new Date('2026-06-30T03:00:00.000Z') }); + writeFileSync(join(dir, 'data.json.gz'), Buffer.from('corrupt')); + + await expect(restoreBackup(name, { config: cfg, log: { info() {}, warn() {}, error() {} } })).rejects.toThrow( + /failed verification/ + ); + }); +}); diff --git a/backend/backup.js b/backend/backup.js new file mode 100644 index 0000000..bd3d0a2 --- /dev/null +++ b/backend/backup.js @@ -0,0 +1,496 @@ +/** + * Database backup automation. + * + * Produces verifiable, rotated snapshots of every persistent data source the + * backend owns: + * - JSON data files (data.json, webhooks.json) + * - The SQL database — SQLite in dev/test, PostgreSQL (via pg_dump) in prod + * + * Each backup is a directory `backups/backup-/` containing one + * gzipped member per source plus a `manifest.json` recording the SHA-256 of + * every source. The manifest is what `verifyBackup()` checks against, so a + * corrupted or truncated member is detected before it is ever relied upon. + * + * The core (create / verify / rotate / schedule) uses only Node built-ins. + * S3 upload (`@aws-sdk/client-s3`), PostgreSQL dumps (`pg_dump`) and the + * SQLite online-backup API (`better-sqlite3`) are loaded lazily and only when + * actually configured, so the module — and its tests — run with zero extra + * dependencies. + * + * Usage: + * node backup.js # create + verify + (optional S3) + rotate + * node backup.js --list # list local backups, newest last + * node backup.js --verify-latest # re-verify the most recent backup + */ + +import { spawn } from 'child_process'; +import { createHash } from 'crypto'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, + copyFileSync, + createWriteStream, + rmSync, +} from 'fs'; +import { gzipSync, gunzipSync } from 'zlib'; +import { basename, dirname, isAbsolute, join, resolve } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ── Logging ───────────────────────────────────────────────────────────────── +const consoleLog = { + info: (m) => console.log(`[backup] ${m}`), + warn: (m) => console.warn(`[backup] ${m}`), + error: (m) => console.error(`[backup] ${m}`), +}; +const silentLog = { info() {}, warn() {}, error() {} }; + +// ── Config ────────────────────────────────────────────────────────────────── +function toInt(value, fallback) { + const n = parseInt(value, 10); + return Number.isFinite(n) ? n : fallback; +} + +function isPostgres(url) { + return typeof url === 'string' && /^postgres(ql)?:\/\//.test(url); +} + +/** + * Build the backup configuration from environment variables, applying the same + * defaults the rest of the backend uses for data-file and database locations. + * + * @param {NodeJS.ProcessEnv} [env] + */ +export function loadBackupConfig(env = process.env) { + const backendDir = __dirname; + + const dataFiles = ( + env.BACKUP_DATA_FILES + ? env.BACKUP_DATA_FILES.split(',').map((s) => s.trim()).filter(Boolean) + : [env.DATA_FILE || 'data.json', env.WEBHOOK_DATA_FILE || 'webhooks.json'] + ).map((f) => (isAbsolute(f) ? f : join(backendDir, f))); + + return { + backendDir, + backupDir: env.BACKUP_DIR ? resolve(env.BACKUP_DIR) : join(backendDir, 'backups'), + dataFiles, + databaseUrl: env.DATABASE_URL || './dev.db', + retentionDays: toInt(env.BACKUP_RETENTION_DAYS, 7), + retentionMax: toInt(env.BACKUP_RETENTION_MAX, 30), + s3: env.BACKUP_S3_BUCKET + ? { + bucket: env.BACKUP_S3_BUCKET, + prefix: (env.BACKUP_S3_PREFIX || 'backups').replace(/\/+$/, ''), + region: env.BACKUP_S3_REGION || env.AWS_REGION || 'us-east-1', + endpoint: env.BACKUP_S3_ENDPOINT || undefined, + } + : null, + }; +} + +// ── Timestamp helpers ───────────────────────────────────────────────────────── +// Backup names embed an ISO timestamp with ':' and '.' replaced by '-' so they +// are valid on every filesystem, e.g. backup-2026-06-30T03-00-00-000Z. +function stampFromDate(d) { + return d.toISOString().replace(/[:.]/g, '-'); +} + +export function parseBackupTimestamp(name) { + const s = name.replace(/^backup-/, ''); + const iso = s.replace(/T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, 'T$1:$2:$3.$4Z'); + const d = new Date(iso); + return Number.isNaN(d.getTime()) ? new Date(0) : d; +} + +// ── Member helpers ──────────────────────────────────────────────────────────── +/** + * Gzip `src` into `dest`, returning the SHA-256 of the *original* bytes plus + * sizes. The checksum is over the uncompressed content so verification also + * proves the gzip round-trips to the exact source. + */ +function gzipFileInto(src, dest) { + const raw = readFileSync(src); + const sha256 = createHash('sha256').update(raw).digest('hex'); + const gz = gzipSync(raw); + writeFileSync(dest, gz); + return { sha256, originalBytes: raw.length, gzipBytes: gz.length }; +} + +// ── Database snapshots ──────────────────────────────────────────────────────── +function pgDump(url, dest, log) { + return new Promise((resolvePromise, reject) => { + const out = createWriteStream(dest); + const proc = spawn('pg_dump', ['--no-owner', '--no-privileges', url], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + proc.stdout.pipe(out); + proc.stderr.on('data', (d) => { + stderr += d.toString(); + }); + proc.on('error', (err) => + reject(new Error(`pg_dump failed to start: ${err.message}. Is the PostgreSQL client installed?`)) + ); + proc.on('close', (code) => { + if (code === 0) { + log.info('pg_dump completed'); + resolvePromise(); + } else { + reject(new Error(`pg_dump exited with code ${code}: ${stderr.trim()}`)); + } + }); + }); +} + +async function sqliteSnapshot(src, dest, log) { + // Prefer better-sqlite3's online backup API for a consistent snapshot of a + // live database; fall back to a plain file copy if it is unavailable. + try { + const { default: Database } = await import('better-sqlite3'); + const db = new Database(src, { readonly: true, fileMustExist: true }); + try { + await db.backup(dest); + } finally { + db.close(); + } + log.info('sqlite online backup completed'); + } catch (err) { + log.warn(`sqlite online backup unavailable (${err.message}); copying file instead`); + copyFileSync(src, dest); + } +} + +async function backupDatabase(config, dir, log) { + const url = config.databaseUrl; + + if (isPostgres(url)) { + const sqlPath = join(dir, 'database.sql'); + await pgDump(url, sqlPath, log); + const member = gzipFileInto(sqlPath, join(dir, 'database.sql.gz')); + rmSync(sqlPath, { force: true }); + return { name: 'database.sql.gz', source: 'database.sql', engine: 'postgres', ...member }; + } + + const file = isAbsolute(url) ? url : join(config.backendDir, url); + if (!existsSync(file)) { + log.info(`skip database (no SQLite file at ${file})`); + return null; + } + const snapshot = join(dir, 'database.sqlite'); + await sqliteSnapshot(file, snapshot, log); + const member = gzipFileInto(snapshot, join(dir, 'database.sqlite.gz')); + rmSync(snapshot, { force: true }); + return { name: 'database.sqlite.gz', source: 'database.sqlite', engine: 'sqlite', ...member }; +} + +// ── Create ──────────────────────────────────────────────────────────────────── +/** + * Create a single backup directory containing gzipped members + manifest. + * Throws if no data source exists, so an empty backup is never recorded. + * + * @param {ReturnType} config + * @param {{ now?: Date, log?: typeof consoleLog }} [opts] + */ +export async function createBackup(config, { now = new Date(), log = silentLog } = {}) { + const name = `backup-${stampFromDate(now)}`; + const dir = join(config.backupDir, name); + mkdirSync(dir, { recursive: true }); + + const members = []; + + for (const file of config.dataFiles) { + if (!existsSync(file)) { + log.info(`skip data file (missing): ${file}`); + continue; + } + const base = basename(file); + const member = gzipFileInto(file, join(dir, `${base}.gz`)); + members.push({ name: `${base}.gz`, source: base, engine: 'json', ...member }); + } + + const dbMember = await backupDatabase(config, dir, log); + if (dbMember) members.push(dbMember); + + if (members.length === 0) { + rmSync(dir, { recursive: true, force: true }); + throw new Error('No data sources found to back up'); + } + + const manifest = { name, version: 1, createdAt: now.toISOString(), members }; + const manifestPath = join(dir, 'manifest.json'); + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + + return { name, dir, manifestPath, manifest }; +} + +// ── Verify ──────────────────────────────────────────────────────────────────── +/** + * Verify a backup directory: every member must exist, gunzip cleanly, and + * match the SHA-256 recorded in the manifest. + * + * @param {string} dir Path to a `backup-` directory. + */ +export async function verifyBackup(dir) { + const manifestPath = join(dir, 'manifest.json'); + if (!existsSync(manifestPath)) throw new Error(`manifest.json not found in ${dir}`); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')); + + const results = (manifest.members || []).map((m) => { + const p = join(dir, m.name); + if (!existsSync(p)) return { name: m.name, ok: false, error: 'missing' }; + try { + const raw = gunzipSync(readFileSync(p)); + const sha = createHash('sha256').update(raw).digest('hex'); + return sha === m.sha256 + ? { name: m.name, ok: true, error: null } + : { name: m.name, ok: false, error: 'checksum mismatch' }; + } catch (err) { + return { name: m.name, ok: false, error: `corrupt gzip: ${err.message}` }; + } + }); + + return { ok: results.length > 0 && results.every((r) => r.ok), results }; +} + +// ── List & rotate ───────────────────────────────────────────────────────────── +/** + * List local backups sorted oldest → newest. + * + * @param {ReturnType} config + */ +export function listBackups(config) { + if (!existsSync(config.backupDir)) return []; + return readdirSync(config.backupDir, { withFileTypes: true }) + .filter((d) => d.isDirectory() && d.name.startsWith('backup-')) + .map((d) => ({ + name: d.name, + dir: join(config.backupDir, d.name), + createdAt: parseBackupTimestamp(d.name), + })) + .sort((a, b) => a.createdAt - b.createdAt); +} + +/** + * Decide which backups violate the retention policy. Pure function — no I/O — + * so the rotation rules can be unit-tested directly. + * + * A backup is expired if it is older than `retentionDays`, OR if keeping it + * would exceed `retentionMax` (the oldest surviving ones are dropped first). + * Setting either limit to 0 disables that rule. + * + * @param {Array<{name:string, dir?:string, createdAt:Date}>} backups + * @param {{retentionDays:number, retentionMax:number}} policy + * @param {Date} [now] + */ +export function selectExpired(backups, { retentionDays, retentionMax }, now = new Date()) { + const sorted = [...backups].sort((a, b) => a.createdAt - b.createdAt); + const expired = new Set(); + + if (retentionDays > 0) { + const cutoff = now.getTime() - retentionDays * 86400000; + for (const b of sorted) if (b.createdAt.getTime() < cutoff) expired.add(b.name); + } + + const survivors = sorted.filter((b) => !expired.has(b.name)); + if (retentionMax > 0 && survivors.length > retentionMax) { + for (const b of survivors.slice(0, survivors.length - retentionMax)) expired.add(b.name); + } + + return sorted.filter((b) => expired.has(b.name)); +} + +/** + * Delete expired local backups per the retention policy. + * + * @returns {string[]} names of removed backups + */ +export function rotateBackups(config, now = new Date(), log = silentLog) { + const expired = selectExpired(listBackups(config), config, now); + for (const b of expired) { + rmSync(b.dir, { recursive: true, force: true }); + log.info(`rotated out: ${b.name}`); + } + return expired.map((b) => b.name); +} + +// ── S3 (second location for disaster recovery) ───────────────────────────────── +async function loadS3() { + try { + return await import('@aws-sdk/client-s3'); + } catch { + throw new Error( + 'BACKUP_S3_BUCKET is set but @aws-sdk/client-s3 is not installed. Run: npm install @aws-sdk/client-s3' + ); + } +} + +function makeS3Client(S3Client, s3) { + return new S3Client({ + region: s3.region, + ...(s3.endpoint ? { endpoint: s3.endpoint, forcePathStyle: true } : {}), + }); +} + +/** + * Upload every file of a backup directory to S3 under `//`. + * Returns the uploaded object keys, or null if S3 is not configured. + */ +export async function uploadBackupToS3(backupDir, config, log = silentLog) { + if (!config.s3) return null; + const { S3Client, PutObjectCommand } = await loadS3(); + const client = makeS3Client(S3Client, config.s3); + + const name = basename(backupDir); + const uploaded = []; + for (const f of readdirSync(backupDir)) { + const Key = `${config.s3.prefix}/${name}/${f}`; + await client.send( + new PutObjectCommand({ Bucket: config.s3.bucket, Key, Body: readFileSync(join(backupDir, f)) }) + ); + uploaded.push(Key); + log.info(`uploaded s3://${config.s3.bucket}/${Key}`); + } + return uploaded; +} + +/** + * Apply the same retention policy to backups stored in S3. + * + * @returns {string[]} names of removed S3 backups + */ +export async function rotateS3Backups(config, now = new Date(), log = silentLog) { + if (!config.s3) return []; + const { S3Client, ListObjectsV2Command, DeleteObjectCommand } = await loadS3(); + const client = makeS3Client(S3Client, config.s3); + + // Discover backup names from object keys: //. + const names = new Map(); + let ContinuationToken; + do { + const page = await client.send( + new ListObjectsV2Command({ + Bucket: config.s3.bucket, + Prefix: `${config.s3.prefix}/`, + ContinuationToken, + }) + ); + for (const obj of page.Contents || []) { + const rest = obj.Key.slice(config.s3.prefix.length + 1); + const backupName = rest.split('/')[0]; + if (!backupName.startsWith('backup-')) continue; + if (!names.has(backupName)) { + names.set(backupName, { name: backupName, createdAt: parseBackupTimestamp(backupName), keys: [] }); + } + names.get(backupName).keys.push(obj.Key); + } + ContinuationToken = page.IsTruncated ? page.NextContinuationToken : undefined; + } while (ContinuationToken); + + const expired = selectExpired([...names.values()], config, now); + for (const b of expired) { + for (const Key of names.get(b.name).keys) { + await client.send(new DeleteObjectCommand({ Bucket: config.s3.bucket, Key })); + } + log.info(`rotated out (s3): ${b.name}`); + } + return expired.map((b) => b.name); +} + +// ── Orchestration ────────────────────────────────────────────────────────────── +/** + * Full backup run: create → verify → upload to S3 (if configured) → rotate + * both local and remote copies. Throws if creation or verification fails so a + * scheduled job / CI step exits non-zero on a bad backup. + */ +export async function runBackup(config = loadBackupConfig(), { now = new Date(), log = consoleLog } = {}) { + log.info('starting backup run'); + const { name, dir } = await createBackup(config, { now, log }); + log.info(`created backup: ${name}`); + + const verification = await verifyBackup(dir); + if (!verification.ok) { + const failed = verification.results.filter((r) => !r.ok).map((r) => `${r.name} (${r.error})`); + throw new Error(`backup verification failed: ${failed.join(', ')}`); + } + log.info(`verification passed (${verification.results.length} member(s))`); + + let s3 = null; + if (config.s3) { + s3 = await uploadBackupToS3(dir, config, log); + await rotateS3Backups(config, now, log); + } + + const rotated = rotateBackups(config, now, log); + log.info(`rotation removed ${rotated.length} old backup(s)`); + + return { name, dir, verification, s3, rotated }; +} + +// ── In-process scheduler (optional, dependency-free) ─────────────────────────── +/** + * Start a recurring in-process backup job when BACKUP_ENABLED=true. Runs every + * BACKUP_INTERVAL_HOURS (default 24). Returns the timer handle, or null when + * disabled. The timer is unref'd so it never keeps the process alive on its own. + * + * For production, an external scheduler (system cron or the GitHub Actions + * workflow in .github/workflows/backup.yml) is recommended — see docs/backups.md. + */ +export function startBackupScheduler({ log = consoleLog } = {}) { + if (process.env.BACKUP_ENABLED !== 'true') return null; + const hours = Math.max(1, toInt(process.env.BACKUP_INTERVAL_HOURS, 24)); + log.info(`backup scheduler enabled: every ${hours}h`); + + const tick = () => { + runBackup(loadBackupConfig(), { log }).catch((err) => + log.error(`scheduled backup failed: ${err.message}`) + ); + }; + + const timer = setInterval(tick, hours * 3600 * 1000); + timer.unref?.(); + return timer; +} + +// ── CLI ───────────────────────────────────────────────────────────────────── +const invokedDirectly = + process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url; + +if (invokedDirectly) { + const args = process.argv.slice(2); + const config = loadBackupConfig(); + + const main = async () => { + if (args.includes('--list')) { + const backups = listBackups(config); + if (backups.length === 0) { + consoleLog.info('no local backups found'); + return; + } + for (const b of backups) consoleLog.info(`${b.name} (${b.createdAt.toISOString()})`); + return; + } + + if (args.includes('--verify-latest')) { + const backups = listBackups(config); + if (backups.length === 0) throw new Error('no local backups to verify'); + const latest = backups[backups.length - 1]; + const v = await verifyBackup(latest.dir); + consoleLog.info(`${latest.name}: ${v.ok ? 'OK' : 'FAILED'}`); + if (!v.ok) throw new Error(JSON.stringify(v.results)); + return; + } + + await runBackup(config, { log: consoleLog }); + }; + + main() + .then(() => process.exit(0)) + .catch((err) => { + consoleLog.error(err.message); + process.exit(1); + }); +} diff --git a/backend/package.json b/backend/package.json index 82789b5..cac9b6a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,7 +12,11 @@ "migrate": "knex migrate:latest", "migrate:rollback": "knex migrate:rollback", "migrate:make": "knex migrate:make", - "migrate:status": "knex migrate:list" + "migrate:status": "knex migrate:list", + "backup": "node backup.js", + "backup:list": "node backup.js --list", + "backup:verify": "node backup.js --verify-latest", + "restore": "node restore.js" }, "jest": { "testEnvironment": "node", diff --git a/backend/restore.js b/backend/restore.js new file mode 100644 index 0000000..55ef4a0 --- /dev/null +++ b/backend/restore.js @@ -0,0 +1,193 @@ +/** + * Restore a backup produced by backup.js. + * + * Restoring verifies the backup first (refusing a corrupt one unless --force), + * then writes each member back to its original location: + * - JSON data files → backend/ + * - SQLite database → the DATABASE_URL path (or backend/database.sqlite) + * - PostgreSQL dump → written to disk; restore is NOT run automatically. + * A PostgreSQL restore overwrites a live database, so the dump is left for + * you to apply with: psql "$DATABASE_URL" < restore-database.sql + * + * Any file that would be overwritten is first moved aside to `.pre-restore` + * so a mistaken restore is always reversible. + * + * Usage: + * node restore.js --list # list available backups + * node restore.js # restore a local backup + * node restore.js --from-s3 # download from S3, then restore + * node restore.js --force # restore even if verification fails + */ + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs'; +import { gunzipSync } from 'zlib'; +import { basename, dirname, isAbsolute, join } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; +import { listBackups, loadBackupConfig, verifyBackup } from './backup.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const consoleLog = { + info: (m) => console.log(`[restore] ${m}`), + warn: (m) => console.warn(`[restore] ${m}`), + error: (m) => console.error(`[restore] ${m}`), +}; +const silentLog = { info() {}, warn() {}, error() {} }; + +function isPostgres(url) { + return typeof url === 'string' && /^postgres(ql)?:\/\//.test(url); +} + +function moveAside(target, log) { + if (existsSync(target)) { + const aside = `${target}.pre-restore`; + renameSync(target, aside); + log.info(`existing file preserved at ${aside}`); + } +} + +function sqliteTarget(config) { + const url = config.databaseUrl; + if (isPostgres(url)) return join(config.backendDir, 'database.sqlite'); + return isAbsolute(url) ? url : join(config.backendDir, url); +} + +/** + * Restore a single backup directory into place. + * + * @param {string} name Backup name, e.g. "backup-2026-06-30T03-00-00-000Z". + * @param {{ config?: ReturnType, log?: typeof consoleLog, force?: boolean }} [opts] + * @returns {Promise} paths written + */ +export async function restoreBackup(name, { config = loadBackupConfig(), log = consoleLog, force = false } = {}) { + const dir = join(config.backupDir, name); + if (!existsSync(dir)) throw new Error(`backup not found: ${dir}`); + + const verification = await verifyBackup(dir); + if (!verification.ok && !force) { + const failed = verification.results.filter((r) => !r.ok).map((r) => `${r.name} (${r.error})`); + throw new Error(`backup failed verification: ${failed.join(', ')}. Re-run with --force to override.`); + } + if (!verification.ok) log.warn('verification failed but --force was given; continuing'); + + const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf-8')); + const written = []; + + for (const m of manifest.members) { + const raw = gunzipSync(readFileSync(join(dir, m.name))); + + if (m.engine === 'postgres') { + const out = join(config.backendDir, 'restore-database.sql'); + writeFileSync(out, raw); + log.warn(`PostgreSQL dump written to ${out}`); + log.warn(`To apply it (this OVERWRITES the database): psql "$DATABASE_URL" < ${out}`); + written.push(out); + continue; + } + + let target; + if (m.engine === 'sqlite') { + target = sqliteTarget(config); + } else { + // JSON data file — restore next to the backend by its original basename. + target = join(config.backendDir, basename(m.source)); + } + + mkdirSync(dirname(target), { recursive: true }); + moveAside(target, log); + writeFileSync(target, raw); + log.info(`restored ${m.name} → ${target}`); + written.push(target); + } + + return written; +} + +// ── S3 download ──────────────────────────────────────────────────────────────── +/** + * Download a backup's objects from S3 into the local backup directory so it can + * be restored. Returns the local directory path. + */ +export async function downloadBackupFromS3(name, config = loadBackupConfig(), log = consoleLog) { + if (!config.s3) throw new Error('S3 is not configured (set BACKUP_S3_BUCKET)'); + let s3mod; + try { + s3mod = await import('@aws-sdk/client-s3'); + } catch { + throw new Error('@aws-sdk/client-s3 is not installed. Run: npm install @aws-sdk/client-s3'); + } + const { S3Client, ListObjectsV2Command, GetObjectCommand } = s3mod; + const client = new S3Client({ + region: config.s3.region, + ...(config.s3.endpoint ? { endpoint: config.s3.endpoint, forcePathStyle: true } : {}), + }); + + const prefix = `${config.s3.prefix}/${name}/`; + const localDir = join(config.backupDir, name); + mkdirSync(localDir, { recursive: true }); + + let ContinuationToken; + let count = 0; + do { + const page = await client.send( + new ListObjectsV2Command({ Bucket: config.s3.bucket, Prefix: prefix, ContinuationToken }) + ); + for (const obj of page.Contents || []) { + const file = obj.Key.slice(prefix.length); + if (!file) continue; + const res = await client.send(new GetObjectCommand({ Bucket: config.s3.bucket, Key: obj.Key })); + const bytes = Buffer.from(await res.Body.transformToByteArray()); + writeFileSync(join(localDir, file), bytes); + count += 1; + log.info(`downloaded ${obj.Key}`); + } + ContinuationToken = page.IsTruncated ? page.NextContinuationToken : undefined; + } while (ContinuationToken); + + if (count === 0) throw new Error(`no objects found in s3://${config.s3.bucket}/${prefix}`); + return localDir; +} + +// ── CLI ───────────────────────────────────────────────────────────────────── +const invokedDirectly = process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url; + +if (invokedDirectly) { + const args = process.argv.slice(2); + const config = loadBackupConfig(); + const force = args.includes('--force'); + + const main = async () => { + if (args.includes('--list')) { + const backups = listBackups(config); + if (backups.length === 0) { + consoleLog.info('no local backups found'); + return; + } + for (const b of backups) consoleLog.info(`${b.name} (${b.createdAt.toISOString()})`); + return; + } + + const fromS3Idx = args.indexOf('--from-s3'); + let name; + if (fromS3Idx !== -1) { + name = args[fromS3Idx + 1]; + if (!name) throw new Error('--from-s3 requires a backup name'); + await downloadBackupFromS3(name, config, consoleLog); + } else { + name = args.find((a) => a.startsWith('backup-')); + if (!name) throw new Error('usage: node restore.js | --from-s3 | --list'); + } + + const written = await restoreBackup(name, { config, log: consoleLog, force }); + consoleLog.info(`restore complete: ${written.length} file(s)`); + }; + + main() + .then(() => process.exit(0)) + .catch((err) => { + consoleLog.error(err.message); + process.exit(1); + }); +} + +export { consoleLog as restoreLogger, silentLog }; diff --git a/docs/backups.md b/docs/backups.md new file mode 100644 index 0000000..169b9dc --- /dev/null +++ b/docs/backups.md @@ -0,0 +1,157 @@ +# Database Backup & Restore + +Automated, verifiable backups of every persistent data source the backend owns, +with offsite copies in S3 and an enforced retention policy. + +## What gets backed up + +| Source | Where it lives | How it's captured | +| --- | --- | --- | +| Asset metadata | `backend/data.json` | gzipped copy | +| Webhooks | `backend/webhooks.json` | gzipped copy | +| SQL database (dev/test) | SQLite file from `DATABASE_URL` (default `dev.db`) | consistent online snapshot via `better-sqlite3`, then gzipped | +| SQL database (prod) | PostgreSQL via `DATABASE_URL` | `pg_dump`, then gzipped | + +Each run produces a directory: + +``` +backend/backups/ +└── backup-2026-06-30T03-00-00-000Z/ + ├── data.json.gz + ├── webhooks.json.gz + ├── database.sqlite.gz # or database.sql.gz for PostgreSQL + └── manifest.json # SHA-256 of each member + metadata +``` + +The `manifest.json` records the SHA-256 of every source file. Verification +re-derives those hashes from the stored copies, so a truncated or corrupted +backup is caught before anyone relies on it. + +## Running a backup manually + +```bash +cd backend +npm run backup # create → verify → upload to S3 (if configured) → rotate +npm run backup:list # list local backups +npm run backup:verify # re-verify the most recent backup +``` + +A backup run exits non-zero if creation or verification fails, so it is safe to +wire into cron or CI. + +## Configuration + +All settings are environment variables (see `backend/.env.example`): + +| Variable | Default | Purpose | +| --- | --- | --- | +| `BACKUP_DIR` | `backend/backups` | local backup location | +| `BACKUP_RETENTION_DAYS` | `7` | delete backups older than N days (0 = disabled) | +| `BACKUP_RETENTION_MAX` | `30` | keep at most N backups (0 = disabled) | +| `BACKUP_DATA_FILES` | `data.json,webhooks.json` | extra files to include | +| `BACKUP_ENABLED` | unset | enable the in-process scheduler | +| `BACKUP_INTERVAL_HOURS` | `24` | scheduler interval | +| `BACKUP_S3_BUCKET` | unset | enable S3 offsite copies | +| `BACKUP_S3_PREFIX` | `backups` | key prefix in the bucket | +| `BACKUP_S3_REGION` | `AWS_REGION` or `us-east-1` | bucket region | +| `BACKUP_S3_ENDPOINT` | unset | S3-compatible endpoint (MinIO, R2, Spaces) | + +## Scheduling backups + +You have three ways to schedule backups; pick whichever fits your deployment. + +### 1. GitHub Actions (recommended for offsite) + +`.github/workflows/backup.yml` runs daily at **03:00 UTC**, uploads to S3, and +attaches the backup to the workflow run as a third copy. Add these repository +secrets: `DATABASE_URL`, `BACKUP_S3_BUCKET`, `AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY` (optionally `BACKUP_S3_PREFIX`, `BACKUP_S3_REGION`). +Trigger an ad-hoc run from the **Actions** tab via *Run workflow*. + +### 2. System cron (on the host running the backend) + +```cron +# Daily backup at 03:00, weekly verification on Sundays. +0 3 * * * cd /app/backend && /usr/bin/npm run backup >> /var/log/rwa-backup.log 2>&1 +0 4 * * 0 cd /app/backend && /usr/bin/npm run backup:verify >> /var/log/rwa-backup.log 2>&1 +``` + +### 3. In-process scheduler + +For platforms without cron, set `BACKUP_ENABLED=true` (and optionally +`BACKUP_INTERVAL_HOURS`). The server then runs a backup on that interval. This +is the simplest option but only runs while the process is up — prefer cron or CI +for production. + +## Offsite storage (S3) + +Set `BACKUP_S3_BUCKET` and install the AWS SDK: + +```bash +cd backend && npm install @aws-sdk/client-s3 +``` + +Each backup is uploaded under `s3:////backup-/`. +Credentials use the standard AWS env vars. S3-compatible stores (MinIO, +Cloudflare R2, DigitalOcean Spaces) work by setting `BACKUP_S3_ENDPOINT`. + +The retention policy is applied to S3 as well as local copies, so old offsite +backups are pruned automatically. + +## Retention / rotation + +After each successful run, backups that are **older than +`BACKUP_RETENTION_DAYS`** OR that push the total **above `BACKUP_RETENTION_MAX`** +(oldest dropped first) are deleted — locally and in S3. The defaults keep one +week of daily backups, capped at 30. Set a limit to `0` to disable it. + +## Restore process + +> ⚠️ Restoring overwrites current data. Any file that would be overwritten is +> first moved aside to `.pre-restore`, so a restore is reversible. + +1. **Find the backup to restore** + + ```bash + cd backend + npm run restore -- --list + ``` + +2. **(If offsite) download from S3 first** — this is done automatically by + `--from-s3`, which fetches the backup into `BACKUP_DIR` and then restores it. + +3. **Restore** + + ```bash + # From a local backup: + node restore.js backup-2026-06-30T03-00-00-000Z + + # From S3: + node restore.js --from-s3 backup-2026-06-30T03-00-00-000Z + + # Restore even if verification fails (last resort): + node restore.js backup-2026-06-30T03-00-00-000Z --force + ``` + + This restores `data.json`, `webhooks.json`, and a SQLite database in place. + +4. **PostgreSQL restore is manual** (it overwrites a live DB, so it is never run + automatically). The dump is written to `backend/restore-database.sql`; apply + it with: + + ```bash + psql "$DATABASE_URL" < backend/restore-database.sql + ``` + +5. **Restart the backend** so it picks up the restored data. + +## Verifying disaster recovery + +Periodically rehearse a restore into a throwaway location to prove backups are +usable: + +```bash +cd backend +npm run backup:verify # checksums of the latest backup +BACKUP_DIR=/tmp/dr-test node restore.js --from-s3 # restore offsite copy +```