feat(rest-client): resilient HTTP client with retry, circuit breaker, and correlation ID

rest-client/README.md

@@ -0,0 +1,108 @@
+# REST API Client Starter - Resilience Patterns
+
+Resilient HTTP client for Node.js services calling external REST APIs. Implements retry, circuit breaker, correlation ID propagation, and response caching.
+
+## Circuit Breaker State Machine
+
+```mermaid
+stateDiagram-v2
+    [*] --> Closed
+
+    Closed --> Open : error rate >= threshold\nAND calls >= volumeThreshold
+    Closed --> Closed : successful request
+
+    Open --> HalfOpen : resetTimeout elapsed
+    Open --> Open : request received (rejected immediately,\nno call to downstream)
+
+    HalfOpen --> Closed : ONE test request succeeds
+    HalfOpen --> Open : ONE test request fails
+```
+
+**Important:** In `HalfOpen` state, only ONE test request is allowed through. This is intentional - the goal is to probe whether the downstream has recovered without sending a flood of traffic. The next request after a half-open success or failure changes state immediately.
+
+## Retry Strategy
+
+Exponential backoff with jitter on retryable status codes:
+- `429 Too Many Requests` - rate limit hit
+- `5xx Server Error` - downstream is failing
+
+Not retried:
+- `4xx` (except 429) - client errors, retrying won't help
+
+```
+Attempt 1: immediate
+Attempt 2: ~200ms + jitter
+Attempt 3: ~400ms + jitter
+Attempt 4: give up, throw
+```
+
+## Correlation ID Propagation
+
+Every outbound request includes `X-Correlation-ID`. The value comes from `AsyncLocalStorage` set by `correlationMiddleware` on inbound requests. If no context is found, a new UUID is generated.
+
+**Express setup:**
+
+```typescript
+import express from 'express';
+import { correlationMiddleware } from './correlation-context';
+
+const app = express();
+app.use(correlationMiddleware); // Must be first middleware
+
+// All downstream HTTP calls from handlers will propagate X-Correlation-ID automatically
+```
+
+Log the correlation ID in every log line for end-to-end request tracing across services.
+
+## Usage
+
+```typescript
+import { ResilientHttpClient } from './resilient-client';
+
+const inventoryClient = new ResilientHttpClient({
+  baseURL: 'https://inventory.internal',
+  timeout: 10_000,
+  circuitBreaker: {
+    errorThresholdPercentage: 50,
+    resetTimeout: 30_000,
+    volumeThreshold: 10, // Don't trip on startup noise
+  },
+  cache: {
+    ttl: 30_000, // Cache GET responses for 30s
+  },
+});
+
+// GET with caching
+const product = await inventoryClient.get<Product>(`/products/${id}`);
+
+// POST (not cached)
+const reservation = await inventoryClient.post<Reservation>('/reservations', {
+  productId: id,
+  quantity: 2,
+});
+
+// Health check endpoint
+app.get('/health', (req, res) => {
+  const circuitOpen = inventoryClient.isCircuitOpen();
+  res.json({ status: circuitOpen ? 'degraded' : 'ok', inventoryCircuit: circuitOpen ? 'open' : 'closed' });
+});
+```
+
+## Configuration Reference
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `timeout` | 10000 | Request timeout in ms |
+| `circuitBreaker.errorThresholdPercentage` | 50 | Error rate % to open circuit |
+| `circuitBreaker.timeout` | 5000 | Request timeout for breaker (ms) |
+| `circuitBreaker.resetTimeout` | 30000 | Time before half-open test (ms) |
+| `circuitBreaker.volumeThreshold` | 10 | Min calls before error % evaluated |
+| `cache.ttl` | 60000 | GET cache TTL in ms |
+
+## Tests
+
+```bash
+npm test
+```
+
+Unit tests cover: successful request, retry on 429, circuit breaker opens at threshold, circuit rejects fast when open, correlation ID injection.

rest-client/nodejs/__tests__/resilient-client.test.ts

@@ -0,0 +1,134 @@
+import { jest } from '@jest/globals';
+
+// Mock axios before importing resilient-client
+const mockRequest = jest.fn();
+
+jest.mock('axios', () => ({
+  default: {
+    create: jest.fn().mockReturnValue({
+      request: mockRequest,
+    }),
+  },
+}));
+
+import { ResilientHttpClient } from '../resilient-client';
+
+describe('ResilientHttpClient', () => {
+  let client: ResilientHttpClient;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    client = new ResilientHttpClient({
+      baseURL: 'https://api.example.com',
+      timeout: 5000,
+      circuitBreaker: {
+        volumeThreshold: 3, // Lower threshold for testing
+        errorThresholdPercentage: 50,
+        resetTimeout: 100, // Short reset for testing
+      },
+    });
+  });
+
+  describe('GET - successful request', () => {
+    it('returns response data on success', async () => {
+      mockRequest.mockResolvedValueOnce({ data: { id: 1, name: 'test' } });
+
+      const result = await client.get<{ id: number; name: string }>('/users/1');
+
+      expect(result).toEqual({ id: 1, name: 'test' });
+      expect(mockRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('uses cached response for identical GET requests', async () => {
+      mockRequest.mockResolvedValue({ data: { id: 1 } });
+
+      await client.get('/users/1');
+      await client.get('/users/1');
+
+      // Second call served from cache
+      expect(mockRequest).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('Retry on 429 and 5xx', () => {
+    it('retries on 429 and succeeds on third attempt', async () => {
+      const rateLimitError = { response: { status: 429 } };
+      mockRequest
+        .mockRejectedValueOnce(rateLimitError)
+        .mockRejectedValueOnce(rateLimitError)
+        .mockResolvedValueOnce({ data: { ok: true } });
+
+      const result = await client.get<{ ok: boolean }>('/resource');
+
+      expect(result).toEqual({ ok: true });
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+    });
+
+    it('retries on 503 and fails after 3 attempts', async () => {
+      const serverError = { response: { status: 503 } };
+      mockRequest.mockRejectedValue(serverError);
+
+      await expect(client.get('/resource')).rejects.toEqual(serverError);
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+    });
+
+    it('does not retry on 400 client error', async () => {
+      const clientError = { response: { status: 400 } };
+      mockRequest.mockRejectedValueOnce(clientError);
+
+      await expect(client.get('/resource')).rejects.toEqual(clientError);
+      expect(mockRequest).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('Circuit breaker', () => {
+    it('opens circuit after failure threshold is reached', async () => {
+      const serverError = { response: { status: 500 } };
+      // Need to exceed volumeThreshold (3) with enough failures to meet errorThresholdPercentage (50%)
+      mockRequest.mockRejectedValue(serverError);
+
+      // Fire enough requests to trip the circuit
+      // volumeThreshold=3, so after 3 calls all failing (100% > 50%), circuit opens
+      const requests = Array.from({ length: 5 }, () =>
+        client.get('/resource').catch(() => undefined),
+      );
+      await Promise.all(requests);
+
+      // Wait a tick for the breaker state to update
+      await new Promise((resolve) => setTimeout(resolve, 50));
+
+      expect(client.isCircuitOpen()).toBe(true);
+    });
+
+    it('rejects fast when circuit is open', async () => {
+      const serverError = { response: { status: 500 } };
+      mockRequest.mockRejectedValue(serverError);
+
+      // Trip the circuit
+      for (let i = 0; i < 5; i++) {
+        await client.get('/resource').catch(() => undefined);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 50));
+
+      // Reset mock to succeed - circuit should still be open and reject fast
+      mockRequest.mockResolvedValue({ data: {} });
+      mockRequest.mockClear();
+
+      await expect(client.get('/resource')).rejects.toThrow();
+      // Should not have called the underlying axios at all (fast rejection)
+      expect(mockRequest).toHaveBeenCalledTimes(0);
+    });
+  });
+
+  describe('Correlation ID propagation', () => {
+    it('injects x-correlation-id header on every request', async () => {
+      mockRequest.mockResolvedValueOnce({ data: {} });
+
+      await client.get('/resource');
+
+      const callArgs = mockRequest.mock.calls[0][0] as { headers: Record<string, string> };
+      expect(callArgs.headers['x-correlation-id']).toBeDefined();
+      expect(typeof callArgs.headers['x-correlation-id']).toBe('string');
+    });
+  });
+});

rest-client/nodejs/correlation-context.ts

@@ -0,0 +1,33 @@
+import { AsyncLocalStorage } from 'async_hooks';
+import { randomUUID } from 'crypto';
+import type { Request, Response, NextFunction } from 'express';
+
+export interface CorrelationContext {
+  correlationId: string;
+}
+
+export const correlationStorage = new AsyncLocalStorage<CorrelationContext>();
+
+/**
+ * Get the current correlation ID from AsyncLocalStorage.
+ * Returns undefined if called outside a request context.
+ */
+export function getCorrelationId(): string | undefined {
+  return correlationStorage.getStore()?.correlationId;
+}
+
+/**
+ * Express middleware that reads X-Correlation-ID from the inbound request.
+ * If not present, generates a new UUID. Stores it in AsyncLocalStorage so
+ * all downstream code in the same request context can read it without threading it manually.
+ */
+export function correlationMiddleware(req: Request, res: Response, next: NextFunction): void {
+  const correlationId =
+    (req.headers['x-correlation-id'] as string | undefined) ?? randomUUID();
+
+  res.setHeader('x-correlation-id', correlationId);
+
+  correlationStorage.run({ correlationId }, () => {
+    next();
+  });
+}