diff --git a/README.md b/README.md index 9e06814..6266bf8 100644 --- a/README.md +++ b/README.md @@ -118,4 +118,12 @@ The app validates all environment variables at startup using [Zod](https://zod.d | `LOG_LEVEL` | No | `info` | `trace` / `debug` / `info` / `warn` / `error` / `fatal` | | `GATEWAY_PROFILING_ENABLED` | No | `false` | Enable request profiling | +## Production Shutdown Expectations + +- The server listens for `SIGTERM` and `SIGINT` and performs a graceful shutdown. +- On shutdown, it stops accepting new HTTP requests, waits for active connections to finish, and closes database resources. +- A 30 second timeout is enforced for in-flight connections; lingering sockets are destroyed to prevent hung termination. +- Shutdown hooks are registered with `process.once(...)` to avoid duplicate execution during restarts. +- The dev workflow (`npm run dev` with `tsx watch`) is preserved. Restarts trigger the same graceful path instead of abrupt termination. + This repo is part of [Callora](https://github.com/your-org/callora). Frontend: `callora-frontend`. Contracts: `callora-contracts`. diff --git a/package-lock.json b/package-lock.json index 54234b2..b9539ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -87,6 +87,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -609,7 +610,8 @@ "version": "0.3.15", "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.3.15.tgz", "integrity": "sha512-Cj++n1Mekf9ETfdc16TlDi+cDDQF0W7EcbyRHYOAeZdsAe8M/FJg18itDTSwyHfar2WIezawM9o0EKaRGVKygQ==", - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/@electric-sql/pglite-socket": { "version": "0.0.20", @@ -1771,6 +1773,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -2217,6 +2220,7 @@ "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*" } @@ -2408,6 +2412,7 @@ "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -2561,6 +2566,7 @@ "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.57.2", "@typescript-eslint/types": "8.57.2", @@ -2799,6 +2805,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3158,6 +3165,7 @@ "integrity": "sha512-yR5HATnqeYNVnkaUTf4bOP2dJSnyhP4puJN/QPRyx4YkBEEUxib422n2XzPqDEHjQQqazoYoADdAm5vE15+dAQ==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" @@ -3294,6 +3302,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3822,8 +3831,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/d": { "version": "1.0.2", @@ -4563,6 +4571,7 @@ "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -5615,6 +5624,7 @@ "integrity": "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -6016,6 +6026,7 @@ "integrity": "sha512-AkXIIFcaazymvey2i/+F94XRnM6TsVLZDhBMLsd1Sf/W0wzsvvpjeyUrCZD6HGG4SDYPgDJDBKeiJTBb10WzMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.3.0", "@jest/types": "30.3.0", @@ -7775,6 +7786,7 @@ "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.15.3.tgz", "integrity": "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg==", "license": "MIT", + "peer": true, "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", @@ -8248,6 +8260,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -8601,6 +8614,7 @@ "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.7.tgz", "integrity": "sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw==", "license": "Unlicense", + "peer": true, "engines": { "node": ">=12" }, @@ -8712,6 +8726,7 @@ "integrity": "sha512-n30qZpWehaYQzigLjmuPisyEsvOzHt7bZeRyg8gZ5DvJo9FGjD+gNaY59Ns3hlLD5/jZH5GBeftIss0jDbUoLg==", "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@prisma/config": "7.5.0", "@prisma/dev": "0.20.0", @@ -9126,8 +9141,7 @@ "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/semver": { "version": "7.7.4", @@ -10182,6 +10196,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -10639,6 +10654,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/src/db.ts b/src/db.ts index 5347dad..ce7003a 100644 --- a/src/db.ts +++ b/src/db.ts @@ -18,6 +18,8 @@ export const pool = new Pool({ connectionTimeoutMillis: config.dbPool.connectionTimeoutMillis, }); +let poolClosed = false; + /** * Convenience helper that proxies to pool.query for simple one-off queries. */ @@ -42,4 +44,12 @@ export async function checkDbHealth(): Promise<{ ok: boolean; error?: string }> error: error instanceof Error ? error.message : 'Unknown database error', }; } +} + +export async function closePgPool(): Promise { + if (poolClosed) { + return; + } + await pool.end(); + poolClosed = true; } \ No newline at end of file diff --git a/src/db/index.ts b/src/db/index.ts index 28e1db3..189e93c 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -5,6 +5,7 @@ import { readFileSync } from 'fs'; import { join } from 'path'; const logger = console; +let sqliteClosed = false; // Create SQLite database instance const sqlite = new Database('./database.db'); @@ -61,7 +62,11 @@ export async function initializeDb() { // Graceful shutdown // Export close function for graceful shutdown -export function closeDb() { +export async function closeDb(): Promise { + if (sqliteClosed) { + return; + } sqlite.close(); + sqliteClosed = true; } export { schema }; diff --git a/src/index.test.ts b/src/index.test.ts index f1fb99c..1bb2b6c 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,6 +1,7 @@ /// import request from 'supertest'; -import app from './index.js'; +import type { Server } from 'http'; +import app, { createGracefulShutdownHandler } from './index.js'; jest.mock('./db/index.js', () => ({ db: {}, @@ -14,3 +15,48 @@ describe('Health API', () => { expect(response.body.status).toBe('ok'); }); }); + +describe('graceful shutdown', () => { + it('closes server and database resources', async () => { + const closeServer = jest.fn((callback: (err?: Error) => void) => callback()); + const closeDatabase = jest.fn(async () => Promise.resolve()); + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() }; + + const shutdown = createGracefulShutdownHandler({ + server: { close: closeServer } as unknown as Server, + activeConnections: new Set(), + closeDatabase, + logger, + timeoutMs: 50, + }); + + await expect(shutdown('SIGTERM')).resolves.toBe(0); + expect(closeServer).toHaveBeenCalledTimes(1); + expect(closeDatabase).toHaveBeenCalledTimes(1); + }); + + it('reuses in-flight shutdown promise on repeated signals', async () => { + let closeCallback: ((err?: Error) => void) | undefined; + const closeServer = jest.fn((callback: (err?: Error) => void) => { + closeCallback = callback; + }); + const closeDatabase = jest.fn(async () => Promise.resolve()); + + const shutdown = createGracefulShutdownHandler({ + server: { close: closeServer } as unknown as Server, + activeConnections: new Set(), + closeDatabase, + timeoutMs: 50, + }); + + const first = shutdown('SIGTERM'); + const second = shutdown('SIGINT'); + + expect(closeServer).toHaveBeenCalledTimes(1); + closeCallback?.(); + + await expect(first).resolves.toBe(0); + await expect(second).resolves.toBe(0); + expect(closeDatabase).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/index.ts b/src/index.ts index 4db6eb4..a26241e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,14 @@ import './config/env.js' import express from 'express'; import { initializeDb, closeDb } from './db/index.js'; -import { type AuthenticatedLocals } from './middleware/requireAuth.js'; +import { closePgPool } from './db.js'; +import { closeDbPool } from './config/health.js'; +import { disconnectPrisma } from './lib/prisma.js'; import { errorHandler } from './middleware/errorHandler.js'; import { createGatewayIpAllowlist } from './middleware/ipAllowlist.js'; import type { Response } from 'express'; import type { Socket } from 'net'; +import type { Server } from 'http'; import { createDeveloperRouter } from './routes/developerRoutes.js'; import { createGatewayRouter } from './routes/gatewayRoutes.js'; @@ -86,6 +89,15 @@ if (isDirectExecution) { const PORT = config.port; + const closeAllDataResources = async () => { + await closeDb(); + await Promise.allSettled([ + closePgPool(), + disconnectPrisma(), + closeDbPool(), + ]); + }; + // Initialize database and start server async function startServer() { try { @@ -103,52 +115,21 @@ if (isDirectExecution) { socket.once('close', () => activeConnections.delete(socket)); }); - async function gracefulShutdown(signal: string) { - console.log(`\n[shutdown] Received ${signal}. Starting graceful shutdown...`); - - // 1. Stop accepting new requests - server.close(() => { - console.log('[shutdown] HTTP server closed. No new requests accepted.'); - }); + const gracefulShutdown = createGracefulShutdownHandler({ + server, + activeConnections, + closeDatabase: closeAllDataResources, + }); - // 2. Wait for in-flight requests to finish (max 30s) - const TIMEOUT_MS = 30_000; - const deadline = setTimeout(() => { - console.warn('[shutdown] Timeout reached. Forcing exit.'); - process.exit(1); - }, TIMEOUT_MS); - deadline.unref(); - - // 3. Wait until all active connections are gone - await new Promise((resolve) => { - if (activeConnections.size === 0) return resolve(); - console.log(`[shutdown] Waiting for ${activeConnections.size} in-flight connection(s)...`); - const interval = setInterval(() => { - if (activeConnections.size === 0) { - clearInterval(interval); - resolve(); - } - }, 200); + const onSignal = (signal: NodeJS.Signals) => { + void gracefulShutdown(signal).then((exitCode) => { + process.exit(exitCode); }); - - // 4. Close the database - console.log('[shutdown] Closing database...'); - try { - closeDb(); - console.log('[shutdown] Database closed.'); - } catch (err) { - console.error('[shutdown] Error closing database:', err); - } - - // 5. Exit cleanly - console.log('[shutdown] Shutdown complete. Exiting.'); - clearTimeout(deadline); - process.exit(0); - } + }; // Register shutdown signals - process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); - process.on('SIGINT', () => gracefulShutdown('SIGINT')); + process.once('SIGTERM', () => onSignal('SIGTERM')); + process.once('SIGINT', () => onSignal('SIGINT')); } catch (error) { console.error('Failed to start server:', error); diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 7239eab..4147552 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -15,6 +15,13 @@ function getPrismaClient(): PrismaClient { return prisma; } +export async function disconnectPrisma(): Promise { + if (!prisma) { + return; + } + await prisma.$disconnect(); +} + export default new Proxy({} as PrismaClient, { get(_target, prop, receiver) { const client = getPrismaClient();