diff --git a/rest-client/README.md b/rest-client/README.md new file mode 100644 index 0000000..817e6f0 --- /dev/null +++ b/rest-client/README.md @@ -0,0 +1,108 @@ +# REST API Client Starter - Resilience Patterns + +Resilient HTTP client for Node.js services calling external REST APIs. Implements retry, circuit breaker, correlation ID propagation, and response caching. + +## Circuit Breaker State Machine + +```mermaid +stateDiagram-v2 + [*] --> Closed + + Closed --> Open : error rate >= threshold\nAND calls >= volumeThreshold + Closed --> Closed : successful request + + Open --> HalfOpen : resetTimeout elapsed + Open --> Open : request received (rejected immediately,\nno call to downstream) + + HalfOpen --> Closed : ONE test request succeeds + HalfOpen --> Open : ONE test request fails +``` + +**Important:** In `HalfOpen` state, only ONE test request is allowed through. This is intentional - the goal is to probe whether the downstream has recovered without sending a flood of traffic. The next request after a half-open success or failure changes state immediately. + +## Retry Strategy + +Exponential backoff with jitter on retryable status codes: +- `429 Too Many Requests` - rate limit hit +- `5xx Server Error` - downstream is failing + +Not retried: +- `4xx` (except 429) - client errors, retrying won't help + +``` +Attempt 1: immediate +Attempt 2: ~200ms + jitter +Attempt 3: ~400ms + jitter +Attempt 4: give up, throw +``` + +## Correlation ID Propagation + +Every outbound request includes `X-Correlation-ID`. The value comes from `AsyncLocalStorage` set by `correlationMiddleware` on inbound requests. If no context is found, a new UUID is generated. + +**Express setup:** + +```typescript +import express from 'express'; +import { correlationMiddleware } from './correlation-context'; + +const app = express(); +app.use(correlationMiddleware); // Must be first middleware + +// All downstream HTTP calls from handlers will propagate X-Correlation-ID automatically +``` + +Log the correlation ID in every log line for end-to-end request tracing across services. + +## Usage + +```typescript +import { ResilientHttpClient } from './resilient-client'; + +const inventoryClient = new ResilientHttpClient({ + baseURL: 'https://inventory.internal', + timeout: 10_000, + circuitBreaker: { + errorThresholdPercentage: 50, + resetTimeout: 30_000, + volumeThreshold: 10, // Don't trip on startup noise + }, + cache: { + ttl: 30_000, // Cache GET responses for 30s + }, +}); + +// GET with caching +const product = await inventoryClient.get(`/products/${id}`); + +// POST (not cached) +const reservation = await inventoryClient.post('/reservations', { + productId: id, + quantity: 2, +}); + +// Health check endpoint +app.get('/health', (req, res) => { + const circuitOpen = inventoryClient.isCircuitOpen(); + res.json({ status: circuitOpen ? 'degraded' : 'ok', inventoryCircuit: circuitOpen ? 'open' : 'closed' }); +}); +``` + +## Configuration Reference + +| Option | Default | Description | +|--------|---------|-------------| +| `timeout` | 10000 | Request timeout in ms | +| `circuitBreaker.errorThresholdPercentage` | 50 | Error rate % to open circuit | +| `circuitBreaker.timeout` | 5000 | Request timeout for breaker (ms) | +| `circuitBreaker.resetTimeout` | 30000 | Time before half-open test (ms) | +| `circuitBreaker.volumeThreshold` | 10 | Min calls before error % evaluated | +| `cache.ttl` | 60000 | GET cache TTL in ms | + +## Tests + +```bash +npm test +``` + +Unit tests cover: successful request, retry on 429, circuit breaker opens at threshold, circuit rejects fast when open, correlation ID injection. diff --git a/rest-client/nodejs/__tests__/resilient-client.test.ts b/rest-client/nodejs/__tests__/resilient-client.test.ts new file mode 100644 index 0000000..1cc4719 --- /dev/null +++ b/rest-client/nodejs/__tests__/resilient-client.test.ts @@ -0,0 +1,134 @@ +import { jest } from '@jest/globals'; + +// Mock axios before importing resilient-client +const mockRequest = jest.fn(); + +jest.mock('axios', () => ({ + default: { + create: jest.fn().mockReturnValue({ + request: mockRequest, + }), + }, +})); + +import { ResilientHttpClient } from '../resilient-client'; + +describe('ResilientHttpClient', () => { + let client: ResilientHttpClient; + + beforeEach(() => { + jest.clearAllMocks(); + client = new ResilientHttpClient({ + baseURL: 'https://api.example.com', + timeout: 5000, + circuitBreaker: { + volumeThreshold: 3, // Lower threshold for testing + errorThresholdPercentage: 50, + resetTimeout: 100, // Short reset for testing + }, + }); + }); + + describe('GET - successful request', () => { + it('returns response data on success', async () => { + mockRequest.mockResolvedValueOnce({ data: { id: 1, name: 'test' } }); + + const result = await client.get<{ id: number; name: string }>('/users/1'); + + expect(result).toEqual({ id: 1, name: 'test' }); + expect(mockRequest).toHaveBeenCalledTimes(1); + }); + + it('uses cached response for identical GET requests', async () => { + mockRequest.mockResolvedValue({ data: { id: 1 } }); + + await client.get('/users/1'); + await client.get('/users/1'); + + // Second call served from cache + expect(mockRequest).toHaveBeenCalledTimes(1); + }); + }); + + describe('Retry on 429 and 5xx', () => { + it('retries on 429 and succeeds on third attempt', async () => { + const rateLimitError = { response: { status: 429 } }; + mockRequest + .mockRejectedValueOnce(rateLimitError) + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce({ data: { ok: true } }); + + const result = await client.get<{ ok: boolean }>('/resource'); + + expect(result).toEqual({ ok: true }); + expect(mockRequest).toHaveBeenCalledTimes(3); + }); + + it('retries on 503 and fails after 3 attempts', async () => { + const serverError = { response: { status: 503 } }; + mockRequest.mockRejectedValue(serverError); + + await expect(client.get('/resource')).rejects.toEqual(serverError); + expect(mockRequest).toHaveBeenCalledTimes(3); + }); + + it('does not retry on 400 client error', async () => { + const clientError = { response: { status: 400 } }; + mockRequest.mockRejectedValueOnce(clientError); + + await expect(client.get('/resource')).rejects.toEqual(clientError); + expect(mockRequest).toHaveBeenCalledTimes(1); + }); + }); + + describe('Circuit breaker', () => { + it('opens circuit after failure threshold is reached', async () => { + const serverError = { response: { status: 500 } }; + // Need to exceed volumeThreshold (3) with enough failures to meet errorThresholdPercentage (50%) + mockRequest.mockRejectedValue(serverError); + + // Fire enough requests to trip the circuit + // volumeThreshold=3, so after 3 calls all failing (100% > 50%), circuit opens + const requests = Array.from({ length: 5 }, () => + client.get('/resource').catch(() => undefined), + ); + await Promise.all(requests); + + // Wait a tick for the breaker state to update + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(client.isCircuitOpen()).toBe(true); + }); + + it('rejects fast when circuit is open', async () => { + const serverError = { response: { status: 500 } }; + mockRequest.mockRejectedValue(serverError); + + // Trip the circuit + for (let i = 0; i < 5; i++) { + await client.get('/resource').catch(() => undefined); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Reset mock to succeed - circuit should still be open and reject fast + mockRequest.mockResolvedValue({ data: {} }); + mockRequest.mockClear(); + + await expect(client.get('/resource')).rejects.toThrow(); + // Should not have called the underlying axios at all (fast rejection) + expect(mockRequest).toHaveBeenCalledTimes(0); + }); + }); + + describe('Correlation ID propagation', () => { + it('injects x-correlation-id header on every request', async () => { + mockRequest.mockResolvedValueOnce({ data: {} }); + + await client.get('/resource'); + + const callArgs = mockRequest.mock.calls[0][0] as { headers: Record }; + expect(callArgs.headers['x-correlation-id']).toBeDefined(); + expect(typeof callArgs.headers['x-correlation-id']).toBe('string'); + }); + }); +}); diff --git a/rest-client/nodejs/correlation-context.ts b/rest-client/nodejs/correlation-context.ts new file mode 100644 index 0000000..bb7569d --- /dev/null +++ b/rest-client/nodejs/correlation-context.ts @@ -0,0 +1,33 @@ +import { AsyncLocalStorage } from 'async_hooks'; +import { randomUUID } from 'crypto'; +import type { Request, Response, NextFunction } from 'express'; + +export interface CorrelationContext { + correlationId: string; +} + +export const correlationStorage = new AsyncLocalStorage(); + +/** + * Get the current correlation ID from AsyncLocalStorage. + * Returns undefined if called outside a request context. + */ +export function getCorrelationId(): string | undefined { + return correlationStorage.getStore()?.correlationId; +} + +/** + * Express middleware that reads X-Correlation-ID from the inbound request. + * If not present, generates a new UUID. Stores it in AsyncLocalStorage so + * all downstream code in the same request context can read it without threading it manually. + */ +export function correlationMiddleware(req: Request, res: Response, next: NextFunction): void { + const correlationId = + (req.headers['x-correlation-id'] as string | undefined) ?? randomUUID(); + + res.setHeader('x-correlation-id', correlationId); + + correlationStorage.run({ correlationId }, () => { + next(); + }); +} diff --git a/rest-client/nodejs/resilient-client.ts b/rest-client/nodejs/resilient-client.ts new file mode 100644 index 0000000..f151d50 --- /dev/null +++ b/rest-client/nodejs/resilient-client.ts @@ -0,0 +1,180 @@ +import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'; +import CircuitBreaker from 'opossum'; +import { randomUUID } from 'crypto'; +import { getCorrelationId } from './correlation-context'; + +export interface ResilientHttpClientOptions { + baseURL: string; + timeout?: number; + circuitBreaker?: { + // Percentage of failures that trips the circuit (default: 50) + errorThresholdPercentage?: number; + // Time in ms a request may take before it's considered a failure (default: 5000) + timeout?: number; + // Time in ms to wait before allowing a test request in half-open state (default: 30000) + resetTimeout?: number; + // Minimum number of calls before error percentage is evaluated (default: 10) + // Setting this too low trips the circuit on startup noise. + volumeThreshold?: number; + }; + cache?: { + // TTL in ms for cached GET responses (default: 60000) + ttl?: number; + }; +} + +interface CacheEntry { + data: unknown; + expiresAt: number; +} + +/** + * Resilient HTTP client with: + * - Exponential backoff + jitter for 429 and 5xx responses (max 3 retries) + * - Circuit breaker via opossum (closed -> open -> half-open) + * - Correlation ID propagation via X-Correlation-ID header + * - GET response caching with TTL + * - OpenTelemetry trace header injection + */ +export class ResilientHttpClient { + private readonly axios: AxiosInstance; + private readonly breaker: CircuitBreaker; + private readonly cache = new Map(); + private readonly cacheTtl: number; + + constructor(options: ResilientHttpClientOptions) { + this.axios = axios.create({ + baseURL: options.baseURL, + timeout: options.timeout ?? 10_000, + }); + + this.cacheTtl = options.cache?.ttl ?? 60_000; + + // Wrap the internal request function with a circuit breaker. + // The breaker transitions: closed (normal) -> open (failing) -> half-open (testing). + this.breaker = new CircuitBreaker(this.executeRequest.bind(this), { + errorThresholdPercentage: options.circuitBreaker?.errorThresholdPercentage ?? 50, + timeout: options.circuitBreaker?.timeout ?? 5_000, + resetTimeout: options.circuitBreaker?.resetTimeout ?? 30_000, + // Minimum calls before the error percentage is evaluated. + // Prevents tripping on 1-2 startup failures before traffic ramps up. + volumeThreshold: options.circuitBreaker?.volumeThreshold ?? 10, + }); + + this.breaker.on('open', () => { + console.warn(`[ResilientHttpClient] Circuit breaker OPEN for ${options.baseURL}`); + }); + this.breaker.on('halfOpen', () => { + console.info(`[ResilientHttpClient] Circuit breaker HALF-OPEN - sending test request`); + }); + this.breaker.on('close', () => { + console.info(`[ResilientHttpClient] Circuit breaker CLOSED - resuming normal traffic`); + }); + } + + async get(path: string, config?: AxiosRequestConfig): Promise { + const cacheKey = `GET:${path}:${JSON.stringify(config?.params ?? {})}`; + const cached = this.cache.get(cacheKey); + + if (cached && cached.expiresAt > Date.now()) { + return cached.data as T; + } + + const response = await this.breaker.fire('GET', path, undefined, config) as AxiosResponse; + + this.cache.set(cacheKey, { data: response.data, expiresAt: Date.now() + this.cacheTtl }); + return response.data; + } + + async post(path: string, body: unknown, config?: AxiosRequestConfig): Promise { + const response = await this.breaker.fire('POST', path, body, config) as AxiosResponse; + return response.data; + } + + async put(path: string, body: unknown, config?: AxiosRequestConfig): Promise { + const response = await this.breaker.fire('PUT', path, body, config) as AxiosResponse; + return response.data; + } + + async delete(path: string, config?: AxiosRequestConfig): Promise { + const response = await this.breaker.fire('DELETE', path, undefined, config) as AxiosResponse; + return response.data; + } + + private async executeRequest( + method: string, + path: string, + body: unknown, + config?: AxiosRequestConfig, + ): Promise { + const correlationId = getCorrelationId() ?? randomUUID(); + + const requestConfig: AxiosRequestConfig = { + ...config, + headers: { + ...config?.headers, + 'x-correlation-id': correlationId, + // OpenTelemetry W3C trace context propagation headers. + // If a trace context exists in the current span, inject it here. + // This is a no-op placeholder - wire in your OTel SDK propagator. + ...(this.getOtelHeaders()), + }, + }; + + return this.withRetry(() => + this.axios.request({ method, url: path, data: body, ...requestConfig }), + ); + } + + /** + * Retry with exponential backoff and jitter for 429 (rate limit) and 5xx (server errors). + * Does not retry 4xx client errors (except 429). + */ + private async withRetry(fn: () => Promise, maxAttempts = 3): Promise { + let lastError: Error | undefined; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await fn(); + } catch (err) { + const status = (err as { response?: { status: number } }).response?.status; + const isRetryable = status === 429 || (status !== undefined && status >= 500); + + if (!isRetryable || attempt === maxAttempts) { + throw err; + } + + lastError = err as Error; + const baseDelay = Math.pow(2, attempt) * 100; + const jitter = Math.random() * baseDelay; + const delay = baseDelay + jitter; + + console.warn( + `[ResilientHttpClient] Attempt ${attempt}/${maxAttempts} failed (${status}), ` + + `retrying in ${Math.round(delay)}ms`, + ); + await sleep(delay); + } + } + + throw lastError; + } + + private getOtelHeaders(): Record { + // Wire in your OpenTelemetry propagator here. + // Example with @opentelemetry/api: + // const carrier: Record = {}; + // propagation.inject(context.active(), carrier); + // return carrier; + return {}; + } + + /** Expose breaker state for health checks */ + isCircuitOpen(): boolean { + return this.breaker.opened; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +}