From cb073ba8590ec18ccb0d5ac6e17048206e03ebf4 Mon Sep 17 00:00:00 2001 From: Abioladory123 Date: Thu, 30 Jul 2026 13:11:30 +0100 Subject: [PATCH 1/3] refactor(backend): remove ts-nocheck from certificate and notification services --- .../src/certificates/CertificateService.ts | 47 +++++--- backend/src/certificates/MetadataGenerator.ts | 57 +++++++-- .../src/notifications/NotificationService.ts | 39 ++++-- .../src/notifications/notification.routes.ts | 97 +++++++++++---- backend/src/types/certificate.types.ts | 2 + .../tests/certificates.metadata.types.test.ts | 111 ++++++++++++++++++ 6 files changed, 290 insertions(+), 63 deletions(-) create mode 100644 backend/tests/certificates.metadata.types.test.ts diff --git a/backend/src/certificates/CertificateService.ts b/backend/src/certificates/CertificateService.ts index f7fb5234..db36518b 100644 --- a/backend/src/certificates/CertificateService.ts +++ b/backend/src/certificates/CertificateService.ts @@ -1,8 +1,15 @@ -// @ts-nocheck +/** + * CertificateService — issuance, verification and reporting for course + * certificates. Fully type-checked: request, response and domain shapes are + * declared explicitly instead of being suppressed. + */ + + import prisma from '../db/index.js'; import { Certificate, CertificateMetadata, + CertificateStatus, MintCertificateRequest, VerificationResult, } from '../types/certificate.types.js'; @@ -90,8 +97,6 @@ export class CertificateService { }, }); - let metadata: CertificateMetadata | undefined; - try { // Generate and pin the certificate image and metadata to decentralized storage const imageBuffer = await certificateImageGenerator.generateCertificateImage({ @@ -110,7 +115,7 @@ export class CertificateService { mimeType: 'image/svg+xml', }); - metadata = this.metadataGenerator.generate(certificate, course, student, { + const metadata = this.metadataGenerator.generate(certificate, course, student, { imageUri: imageAsset.ipfsUri, externalUrl: `${process.env.API_BASE_URL || 'http://localhost:8080'}/api/v1/certificates/${ certificate.tokenId || tokenIdValue @@ -119,7 +124,7 @@ export class CertificateService { const metadataAsset = await storageService.pinCertificateMetadata({ certificateId: certificateId, - content: metadata, + content: { ...metadata }, }); // Call blockchain service to mint actual NFT @@ -140,7 +145,7 @@ export class CertificateService { // Update returned certificate certificate.certificateHash = mintResult.transactionHash; certificate.contractAddress = mintResult.contractAddress; - certificate.status = 'ACTIVE' as any; + certificate.status = CertificateStatus.ACTIVE; certificate.tokenId = mintResult.tokenId || tokenIdValue; logger.info(`Certificate minted on-chain: ${certificateId} -> token ${mintResult.tokenId}`, { @@ -148,6 +153,9 @@ export class CertificateService { tokenId: mintResult.tokenId, txHash: mintResult.transactionHash, }); + + // Return certificate with metadata + return { ...certificate, metadata }; } catch (error) { logger.error(`Certificate issuance failed for ${certificateId}:`, error); await prisma.certificate.update({ @@ -161,9 +169,6 @@ export class CertificateService { `Failed to mint certificate: ${error instanceof Error ? error.message : 'Unknown error'}` ); } - - // Return certificate with metadata - return { ...certificate, metadata }; } /** @@ -226,7 +231,7 @@ export class CertificateService { return { isValid: false, certificate: null, - status: 'invalid' as any, + status: CertificateStatus.INVALID, onChainData: null, message: 'Certificate not found', }; @@ -250,13 +255,13 @@ export class CertificateService { }; const result: VerificationResult = { - isValid: certificate.status === 'ACTIVE', + isValid: certificate.status === CertificateStatus.ACTIVE, certificate: metadata, - status: certificate.status as any, + status: this.toCertificateStatus(certificate.status), onChainData, }; - if (certificate.status === 'REVOKED') { + if (certificate.status === CertificateStatus.REVOKED) { result.revocationInfo = { revokedAt: certificate.revokedAt!, reason: certificate.revocationReason!, @@ -305,7 +310,7 @@ export class CertificateService { results.push({ isValid: false, certificate: null, - status: 'invalid' as any, + status: CertificateStatus.INVALID, onChainData: null, message: 'Certificate not found', }); @@ -326,9 +331,9 @@ export class CertificateService { }; results.push({ - isValid: cert.status === 'ACTIVE', + isValid: cert.status === CertificateStatus.ACTIVE, certificate: metadata, - status: cert.status as any, + status: this.toCertificateStatus(cert.status), onChainData, }); } @@ -552,6 +557,16 @@ export class CertificateService { }); } + /** + * Narrows a persisted status string to the CertificateStatus union. + * Unknown values (legacy rows, manual edits) resolve to INVALID rather + * than being cast blindly. + */ + private toCertificateStatus(status: string): CertificateStatus { + const known = Object.values(CertificateStatus) as string[]; + return known.includes(status) ? (status as CertificateStatus) : CertificateStatus.INVALID; + } + /** * Extracts wallet address from DID string */ diff --git a/backend/src/certificates/MetadataGenerator.ts b/backend/src/certificates/MetadataGenerator.ts index ec1f26d0..e15f1945 100644 --- a/backend/src/certificates/MetadataGenerator.ts +++ b/backend/src/certificates/MetadataGenerator.ts @@ -7,6 +7,29 @@ import { import { Certificate } from '@prisma/client'; import { API_BASE_URL, ISSUER_NAME, ISSUER_DID } from '../config/rpcConfig.js'; +/** + * Minimal course shape required to build certificate metadata. + * Kept structural so both full Prisma rows and narrowed `select` + * projections satisfy it. + */ +export interface MetadataCourseInput { + /** Course identifier as stored. */ + id: string; + title: string; + instructor: string; + credits: number; +} + +/** + * Minimal student shape required to build certificate metadata. + * Email and other PII are deliberately absent — metadata is public. + */ +export interface MetadataStudentInput { + firstName?: string | null; + lastName?: string | null; + walletAddress?: string | null; +} + export class MetadataGenerator { private readonly baseUrl: string; private readonly issuerName: string; @@ -22,9 +45,9 @@ export class MetadataGenerator { * Generates complete NFT-compliant certificate metadata */ generate( - certificate: Certificate & { student: any; course: any }, - course: any, - student: any, + certificate: Certificate, + course: MetadataCourseInput, + student: MetadataStudentInput, options: { imageUri?: string; externalUrl?: string } = {} ): CertificateMetadata { // Build verification info @@ -84,7 +107,10 @@ export class MetadataGenerator { /** * Builds course information object */ - private buildCourseInfo(course: any, certificate: Certificate): CertificateCourseInfo { + private buildCourseInfo( + course: MetadataCourseInput, + certificate: Certificate + ): CertificateCourseInfo { const dateStr = certificate.issuedAt.toISOString().split('T')[0] || ''; return { id: course.id, @@ -99,7 +125,10 @@ export class MetadataGenerator { /** * Builds student information object (privacy-aware, no email) */ - private buildStudentInfo(student: any, certificate: Certificate): CertificateStudentInfo { + private buildStudentInfo( + student: MetadataStudentInput, + certificate: Certificate + ): CertificateStudentInfo { const fullName = `${student.firstName || ''} ${student.lastName || ''}`.trim(); const walletAddress = student.walletAddress || this.extractWalletFromDid(certificate.did); @@ -126,7 +155,11 @@ export class MetadataGenerator { /** * Builds certificate display name */ - private buildCertificateName(certificate: Certificate, course: any, student: any): string { + private buildCertificateName( + certificate: Certificate, + course: MetadataCourseInput, + student: MetadataStudentInput + ): string { const courseName = course.title; const studentName = `${student.firstName || ''} ${student.lastName || ''}`.trim(); @@ -136,7 +169,11 @@ export class MetadataGenerator { /** * Builds certificate description */ - private buildCertificateDescription(certificate: Certificate, course: any, student: any): string { + private buildCertificateDescription( + certificate: Certificate, + course: MetadataCourseInput, + student: MetadataStudentInput + ): string { const studentName = `${student.firstName || ''} ${student.lastName || ''}`.trim(); const completionDate = certificate.issuedAt.toLocaleDateString('en-US', { year: 'numeric', @@ -159,8 +196,8 @@ export class MetadataGenerator { */ private buildAttributes( certificate: Certificate, - course: any, - student: any + course: MetadataCourseInput, + student: MetadataStudentInput ): Array<{ trait_type: string; value: string | number }> { const attributes = [ { @@ -177,7 +214,7 @@ export class MetadataGenerator { }, { trait_type: 'Completion Date', - value: certificate.issuedAt.toISOString().split('T')[0], + value: certificate.issuedAt.toISOString().split('T')[0] ?? '', }, { trait_type: 'Certificate ID', diff --git a/backend/src/notifications/NotificationService.ts b/backend/src/notifications/NotificationService.ts index 2c737f3a..43cc9d14 100644 --- a/backend/src/notifications/NotificationService.ts +++ b/backend/src/notifications/NotificationService.ts @@ -1,4 +1,4 @@ -// @ts-nocheck +import redisClient from '../cache/RedisClient.js'; import logger from '../utils/logger.js'; import { CourseNotification, @@ -66,9 +66,13 @@ export async function createNotification( existing.unshift(notification); store.set(key, existing); - // Broadcast so all server instances & connected WebSocket clients receive it + // Broadcast so all server instances & connected WebSocket clients receive it. + // No pub client means Redis is unavailable — the local store still holds it. try { - await pubClient.publish('course_notifications', JSON.stringify(notification)); + const pubClient = redisClient.getPubClient(); + if (pubClient) { + await pubClient.publish('course_notifications', JSON.stringify(notification)); + } } catch (err) { logger.warn('Failed to publish course_notification to Redis:', err); } @@ -104,8 +108,9 @@ export function getNotifications(userId: string): NotificationListResponse { export function markAsRead(notificationId: string): boolean { for (const [, notifications] of store.entries()) { const idx = notifications.findIndex((n) => n.id === notificationId); - if (idx !== -1) { - notifications[idx] = { ...notifications[idx], read: true }; + const target = idx === -1 ? undefined : notifications[idx]; + if (target) { + notifications[idx] = { ...target, read: true }; return true; } } @@ -125,8 +130,9 @@ export function markAllAsRead(userId: string): number { const notifs = store.get(key); if (notifs) { for (let i = 0; i < notifs.length; i++) { - if (!notifs[i].read) { - notifs[i] = { ...notifs[i], read: true }; + const current = notifs[i]; + if (current && !current.read) { + notifs[i] = { ...current, read: true }; count++; } } @@ -151,12 +157,21 @@ function mergeSorted( let i = 0; let j = 0; while (result.length < max && (i < a.length || j < b.length)) { - if (i >= a.length) { - result.push(b[j++]); - } else if (j >= b.length) { - result.push(a[i++]); + const left = a[i]; + const right = b[j]; + + if (!left) { + j++; + if (right) result.push(right); + } else if (!right) { + i++; + result.push(left); + } else if (left.createdAt >= right.createdAt) { + i++; + result.push(left); } else { - result.push(a[i].createdAt >= b[j].createdAt ? a[i++] : b[j++]); + j++; + result.push(right); } } return result; diff --git a/backend/src/notifications/notification.routes.ts b/backend/src/notifications/notification.routes.ts index 8683b2ac..a42c98a8 100644 --- a/backend/src/notifications/notification.routes.ts +++ b/backend/src/notifications/notification.routes.ts @@ -1,13 +1,42 @@ -// @ts-nocheck import { Router, Request, Response } from 'express'; import { getNotifications, markAsRead, markAllAsRead, } from './NotificationService.js'; +import { NotificationListResponse } from './notification.types.js'; const router = Router(); +/** Query string accepted by `GET /api/notifications`. */ +interface ListNotificationsQuery { + userId?: string; +} + +/** Route params for `PUT /api/notifications/:id/read`. */ +interface MarkAsReadParams { + id: string; +} + +/** Body accepted by `PUT /api/notifications/read-all`. */ +interface MarkAllAsReadBody { + userId?: string; +} + +/** Error payload returned by the notification routes on 400/404. */ +interface NotificationErrorResponse { + error: string; +} + +/** Success payload for the two mutation routes. */ +interface MarkAsReadResponse { + success: true; +} + +interface MarkAllAsReadResponse extends MarkAsReadResponse { + updatedCount: number; +} + /** * GET /api/notifications * @@ -19,16 +48,22 @@ const router = Router(); * 200 - { notifications, total, unreadCount } * 400 - Missing userId parameter */ -router.get('/', (req: Request, res: Response) => { - const userId = req.query.userId as string | undefined; +router.get( + '/', + ( + req: Request, + res: Response + ) => { + const { userId } = req.query; - if (!userId) { - return res.status(400).json({ error: 'Missing required query parameter: userId' }); - } + if (!userId) { + return res.status(400).json({ error: 'Missing required query parameter: userId' }); + } - const result = getNotifications(userId); - return res.json(result); -}); + const result = getNotifications(userId); + return res.json(result); + } +); /** * PUT /api/notifications/:id/read @@ -39,16 +74,22 @@ router.get('/', (req: Request, res: Response) => { * 200 - { success: true } * 404 - Notification not found */ -router.put('/:id/read', (req: Request, res: Response) => { - const { id } = req.params; - const found = markAsRead(id); +router.put( + '/:id/read', + ( + req: Request, + res: Response + ) => { + const { id } = req.params; + const found = markAsRead(id); - if (!found) { - return res.status(404).json({ error: 'Notification not found' }); - } + if (!found) { + return res.status(404).json({ error: 'Notification not found' }); + } - return res.json({ success: true }); -}); + return res.json({ success: true }); + } +); /** * PUT /api/notifications/read-all @@ -61,15 +102,21 @@ router.put('/:id/read', (req: Request, res: Response) => { * 200 - { success: true, updatedCount: number } * 400 - Missing userId in body */ -router.put('/read-all', (req: Request, res: Response) => { - const { userId } = req.body as { userId?: string }; +router.put( + '/read-all', + ( + req: Request, + res: Response + ) => { + const { userId } = req.body; - if (!userId) { - return res.status(400).json({ error: 'Missing required field: userId' }); - } + if (!userId) { + return res.status(400).json({ error: 'Missing required field: userId' }); + } - const updatedCount = markAllAsRead(userId); - return res.json({ success: true, updatedCount }); -}); + const updatedCount = markAllAsRead(userId); + return res.json({ success: true, updatedCount }); + } +); export default router; diff --git a/backend/src/types/certificate.types.ts b/backend/src/types/certificate.types.ts index e9e797db..ad7379e7 100644 --- a/backend/src/types/certificate.types.ts +++ b/backend/src/types/certificate.types.ts @@ -51,6 +51,8 @@ export enum CertificateStatus { EXPIRED = 'EXPIRED', PENDING = 'PENDING', FAILED = 'FAILED', + /** Not a persisted status — returned when a certificate cannot be found. */ + INVALID = 'invalid', } // Certificate entity with DB fields - matches Prisma output diff --git a/backend/tests/certificates.metadata.types.test.ts b/backend/tests/certificates.metadata.types.test.ts new file mode 100644 index 00000000..95318555 --- /dev/null +++ b/backend/tests/certificates.metadata.types.test.ts @@ -0,0 +1,111 @@ +/** + * Type-boundary tests for the certificate metadata generator. + * + * These cover the shapes that CertificateService passes into + * MetadataGenerator now that the module is fully type-checked: + * - a full Prisma Certificate row + * - a narrowed `select` projection of Student (no email / PII) + */ + +import type { Certificate } from '@prisma/client'; +import { + MetadataGenerator, + MetadataCourseInput, + MetadataStudentInput, +} from '../src/certificates/MetadataGenerator.js'; + +const generator = new MetadataGenerator(); + +const issuedAt = new Date('2026-01-15T10:00:00.000Z'); + +const certificate: Certificate = { + id: 'cert-1', + workspaceId: 'default', + studentId: 'student-1', + courseId: 'course-1', + tokenId: '4242', + issuedAt, + certificateHash: 'tx-hash', + status: 'ACTIVE', + did: 'did:stellar:GCERTSTUDENTWALLET', + metadataUri: null, + contractAddress: 'CCONTRACT', + network: 'stellar-testnet', + grade: 'A', + revokedAt: null, + revocationReason: null, + revokedBy: null, + previousVersionId: null, + transactionHash: 'tx-hash', + createdAt: issuedAt, + updatedAt: issuedAt, +}; + +const course: MetadataCourseInput = { + id: 'course-1', + title: 'Soroban Smart Contracts', + instructor: 'Ada Lovelace', + credits: 4, +}; + +describe('MetadataGenerator type boundaries', () => { + it('builds metadata from a full Prisma row and a narrowed student projection', () => { + const student: MetadataStudentInput = { + firstName: 'Grace', + lastName: 'Hopper', + walletAddress: 'GWALLETADDRESS', + }; + + const metadata = generator.generate(certificate, course, student); + + expect(metadata.name).toBe('Grace Hopper - Soroban Smart Contracts Certificate'); + expect(metadata.course).toEqual({ + id: 'course-1', + title: 'Soroban Smart Contracts', + instructor: 'Ada Lovelace', + credits: 4, + completionDate: '2026-01-15', + grade: 'A', + }); + expect(metadata.student).toEqual({ + name: 'Grace Hopper', + walletAddress: 'GWALLETADDRESS', + }); + expect(metadata.verification.tokenId).toBe('4242'); + expect(metadata.verification.contractAddress).toBe('CCONTRACT'); + }); + + it('falls back to the certificate DID when the student has no wallet address', () => { + const student: MetadataStudentInput = { + firstName: null, + lastName: null, + walletAddress: null, + }; + + const metadata = generator.generate(certificate, course, student); + + expect(metadata.student.walletAddress).toBe('GCERTSTUDENTWALLET'); + expect(metadata.student.name).toBe('Web3 Student'); + }); + + it('honours explicit image and external URL overrides', () => { + const metadata = generator.generate( + certificate, + course, + { firstName: 'Grace', lastName: 'Hopper' }, + { imageUri: 'ipfs://image-cid', externalUrl: 'https://example.test/cert/4242' } + ); + + expect(metadata.image).toBe('ipfs://image-cid'); + expect(metadata.external_url).toBe('https://example.test/cert/4242'); + }); + + it('exposes completion date and grade as NFT attributes', () => { + const metadata = generator.generate(certificate, course, { firstName: 'Grace' }); + const traits = new Map(metadata.attributes.map((a) => [a.trait_type, a.value])); + + expect(traits.get('Completion Date')).toBe('2026-01-15'); + expect(traits.get('Grade')).toBe('A'); + expect(traits.get('Credits')).toBe(4); + }); +}); From 78af16ffa8c5b1f83b1b00b07a70b29aec92b70e Mon Sep 17 00:00:00 2001 From: Abioladory123 Date: Thu, 30 Jul 2026 13:11:37 +0100 Subject: [PATCH 2/3] feat(backend): add versioned API error envelope with correlation IDs --- backend/API_ERROR_CONTRACT.md | 68 ++++++ backend/src/config/swagger.ts | 132 +++++++++++ backend/src/middleware/errorHandler.ts | 21 +- backend/src/middleware/validation.ts | 36 +-- backend/src/utils/apiError.ts | 260 +++++++++++++++++++++ backend/tests/certificates.routes.test.ts | 13 +- backend/tests/errorEnvelope.routes.test.ts | 99 ++++++++ backend/tests/sentry.errorHandler.test.ts | 87 ++++++- 8 files changed, 680 insertions(+), 36 deletions(-) create mode 100644 backend/API_ERROR_CONTRACT.md create mode 100644 backend/src/utils/apiError.ts create mode 100644 backend/tests/errorEnvelope.routes.test.ts diff --git a/backend/API_ERROR_CONTRACT.md b/backend/API_ERROR_CONTRACT.md new file mode 100644 index 00000000..d964f20d --- /dev/null +++ b/backend/API_ERROR_CONTRACT.md @@ -0,0 +1,68 @@ +# API Error Contract + +Every handled error returned by the backend uses one envelope. The shape is +declared in `src/utils/apiError.ts` and published in OpenAPI as +`components.schemas.ErrorEnvelope` (see `/api-docs`). + +## Envelope + +```json +{ + "error": { + "version": "1", + "code": "VALIDATION_FAILED", + "message": "Request validation failed", + "requestId": "9f1c2e3a-6b74-4c0f-9a5c-7b1d2e3f4a5b", + "timestamp": "2026-01-01T12:00:00.000Z", + "fieldErrors": [{ "field": "tokenId", "message": "tokenId must be alphanumeric" }] + } +} +``` + +| Field | Always present | Notes | +| --- | --- | --- | +| `version` | yes | Envelope schema version. Bumped only on a breaking change. | +| `code` | yes | Stable machine-readable code — branch on this, never on `message`. | +| `message` | yes | Client-safe text. Server faults collapse to a generic sentence. | +| `requestId` | yes | Correlation ID; also returned as the `X-Correlation-ID` header. | +| `timestamp` | yes | ISO 8601, server clock. | +| `fieldErrors` | no | Present on validation failures. Field path + reason only — never the submitted value. | + +## Codes + +`BAD_REQUEST`, `VALIDATION_FAILED`, `UNAUTHORIZED`, `FORBIDDEN`, `NOT_FOUND`, +`CONFLICT`, `UNPROCESSABLE_ENTITY`, `RATE_LIMITED`, `INTERNAL_ERROR`, +`SERVICE_UNAVAILABLE`. + +## Raising errors + +```ts +import { ApiError } from '../utils/apiError.js'; + +throw ApiError.notFound('Certificate not found'); +throw ApiError.validationFailed('Request validation failed', [ + { field: 'grade', message: 'grade must be one of A–F' }, +]); +throw ApiError.internal(); // message is replaced with the generic sentence +``` + +Anything else that reaches the global handler (`src/middleware/errorHandler.ts`) +becomes a 500 `INTERNAL_ERROR`. Zod failures raised by +`src/middleware/validation.ts` become 400 `VALIDATION_FAILED` with `fieldErrors`. + +## Client messages vs server logs + +Stack traces and raw error messages never leave the process. For every error the +handler writes a log entry containing `requestId`, `code`, `statusCode`, the raw +message, the stack and the request method/path — 5xx at `error` level, 4xx at +`warn`. To investigate a report, take the `requestId` the client saw and search +the logs for it. + +`requestId` resolution order: the ID assigned by `detailedRequestLogger`, then an +inbound `X-Correlation-ID` or `X-Request-ID` header, then a freshly generated +UUID — so an error response is never returned without one. + +## Tests + +- `tests/errorEnvelope.routes.test.ts` — route-level contract (validation, 404, 500, correlation ID echo). +- `tests/sentry.errorHandler.test.ts` — global handler unit behaviour. diff --git a/backend/src/config/swagger.ts b/backend/src/config/swagger.ts index f80c56e9..07762bc3 100644 --- a/backend/src/config/swagger.ts +++ b/backend/src/config/swagger.ts @@ -29,6 +29,138 @@ const options: swaggerJsdoc.Options = { scheme: 'bearer', bearerFormat: 'JWT', }, + metricsToken: { + type: 'apiKey', + in: 'header', + name: 'X-Metrics-Token', + description: 'Shared secret for operational metrics endpoints (METRICS_AUTH_TOKEN).', + }, + }, + schemas: { + ApiFieldError: { + type: 'object', + description: 'A single rejected field. Never contains the submitted value.', + required: ['field', 'message'], + properties: { + field: { + type: 'string', + description: 'Dot-separated path of the invalid field.', + example: 'tokenId', + }, + message: { + type: 'string', + description: 'Reason the field was rejected.', + example: 'tokenId must be alphanumeric', + }, + }, + }, + ErrorEnvelope: { + type: 'object', + description: + 'Single error envelope used by every handled error response. `message` is always safe for clients; full detail is logged server-side against `requestId`.', + required: ['error'], + properties: { + error: { + type: 'object', + required: ['version', 'code', 'message', 'requestId', 'timestamp'], + properties: { + version: { + type: 'string', + description: 'Envelope schema version. Bumped only on breaking changes.', + example: '1', + }, + code: { + type: 'string', + description: 'Stable machine-readable error code. Branch on this, not on text.', + enum: [ + 'BAD_REQUEST', + 'VALIDATION_FAILED', + 'UNAUTHORIZED', + 'FORBIDDEN', + 'NOT_FOUND', + 'CONFLICT', + 'UNPROCESSABLE_ENTITY', + 'RATE_LIMITED', + 'INTERNAL_ERROR', + 'SERVICE_UNAVAILABLE', + ], + example: 'VALIDATION_FAILED', + }, + message: { + type: 'string', + description: + 'Client-safe description. Server faults collapse to a generic sentence; stack traces are never included.', + example: 'Request validation failed', + }, + requestId: { + type: 'string', + description: + 'Correlation ID for this request; also returned as the X-Correlation-ID response header. Quote it in support requests.', + example: '9f1c2e3a-6b74-4c0f-9a5c-7b1d2e3f4a5b', + }, + timestamp: { + type: 'string', + format: 'date-time', + }, + fieldErrors: { + type: 'array', + description: 'Present on validation failures only.', + items: { $ref: '#/components/schemas/ApiFieldError' }, + }, + }, + }, + }, + }, + }, + responses: { + BadRequest: { + description: 'Malformed request.', + headers: { + 'X-Correlation-ID': { + description: 'Correlation ID matching error.requestId.', + schema: { type: 'string' }, + }, + }, + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + ValidationError: { + description: 'Request validation failed — see error.fieldErrors.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + Unauthorized: { + description: 'Missing or invalid credentials.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + Forbidden: { + description: 'Authenticated but not permitted.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + NotFound: { + description: 'Resource not found.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + RateLimited: { + description: 'Rate limit exceeded.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, + InternalError: { + description: 'Unexpected server error. Detail is logged against error.requestId.', + content: { + 'application/json': { schema: { $ref: '#/components/schemas/ErrorEnvelope' } }, + }, + }, }, }, security: [ diff --git a/backend/src/middleware/errorHandler.ts b/backend/src/middleware/errorHandler.ts index dce5d0c3..47dec243 100644 --- a/backend/src/middleware/errorHandler.ts +++ b/backend/src/middleware/errorHandler.ts @@ -1,6 +1,8 @@ import { Request, Response, NextFunction } from 'express'; import { captureException } from '../utils/sentry.js'; +import { sendErrorEnvelope } from '../utils/apiError.js'; +/** Wraps an async handler so rejections reach the global error handler. */ export const asyncHandler = ( fn: (req: Request, res: Response, next: NextFunction) => Promise ) => { @@ -9,12 +11,19 @@ export const asyncHandler = ( }; }; -// Global error handler middleware +/** + * Global error handler — emits the versioned error envelope documented in + * `src/utils/apiError.ts`. Client messages stay safe (5xx is collapsed to a + * generic sentence); the full error and stack are logged against the same + * correlation ID that is returned to the caller. + */ export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { captureException(err); - console.error('Error:', err instanceof Error ? err.stack || err.message : err); - res.status(500).json({ - status: 'error', - message: 'Internal server error', - }); + + // Headers already flushed — nothing valid can be sent, hand back to Express. + if (res.headersSent) { + return next(err); + } + + return sendErrorEnvelope(req, res, err); }; diff --git a/backend/src/middleware/validation.ts b/backend/src/middleware/validation.ts index 5f159085..2322cd6f 100644 --- a/backend/src/middleware/validation.ts +++ b/backend/src/middleware/validation.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from 'express'; import { z } from 'zod'; -import { ApiResponse } from '../utils/response.js'; +import { ApiError, ApiFieldError, sendErrorEnvelope } from '../utils/apiError.js'; // Subscription creation validation schema export const subscriptionCreateSchema = z.object({ @@ -22,30 +22,36 @@ export const subscriptionUpdateSchema = z.object({ isActive: z.boolean(), }); -// Validation middleware factory +/** + * Map Zod issues to envelope field errors. + * Only the field path and the reason are exposed — never the submitted value. + */ +export const toFieldErrors = (error: z.ZodError): ApiFieldError[] => + error.issues.map((issue: z.ZodIssue) => ({ + field: issue.path.join('.') || '(root)', + message: issue.message, + })); + +// Validation middleware factory — emits the versioned error envelope. export const validate = (schema: z.ZodSchema) => { return (req: Request, res: Response, next: NextFunction) => { try { + // Route params and body are validated together; the merged result + // replaces req.body so handlers read one typed object. const validatedData = schema.parse({ ...req.params, ...req.body }); req.body = validatedData; next(); } catch (error) { if (error instanceof z.ZodError) { - const errorMessages = error.issues.map((err: z.ZodIssue) => ({ - field: err.path.join('.'), - message: err.message, - })); - - return res.status(400).json({ - ...ApiResponse.error('Validation failed', errorMessages), - error: errorMessages.map((e) => `${e.field}: ${e.message}`).join(', '), - }); + return sendErrorEnvelope( + req, + res, + ApiError.validationFailed('Request validation failed', toFieldErrors(error)) + ); } - return res.status(500).json({ - ...ApiResponse.error('Internal server error'), - error: error instanceof Error ? error.message : 'Internal server error', - }); + // Unexpected failure inside the schema itself — never leak the detail. + return sendErrorEnvelope(req, res, ApiError.internal(undefined, error)); } }; }; diff --git a/backend/src/utils/apiError.ts b/backend/src/utils/apiError.ts new file mode 100644 index 00000000..499ad0b0 --- /dev/null +++ b/backend/src/utils/apiError.ts @@ -0,0 +1,260 @@ +/** + * Versioned API error envelope. + * + * Every handled error leaving the API is serialised through this module so + * clients (students, the frontend, integrators) see exactly one shape: + * + * { + * "error": { + * "version": "1", + * "code": "VALIDATION_FAILED", + * "message": "Request validation failed", + * "requestId": "e1f0…", + * "timestamp": "2026-01-01T00:00:00.000Z", + * "fieldErrors": [{ "field": "tier", "message": "Invalid enum value" }] + * } + * } + * + * The `message` is always safe for clients: internal failures are reduced to a + * generic sentence, while the full error (stack included) is written to the + * server log under the same `requestId` so the two can be correlated. + */ + +import type { Request, Response } from 'express'; +import { randomUUID } from 'node:crypto'; +import logger from './logger.js'; + +/** Envelope schema version. Bump only on a breaking shape change. */ +export const ERROR_ENVELOPE_VERSION = '1'; + +/** Message returned to clients whenever the cause is a server-side fault. */ +export const GENERIC_SERVER_MESSAGE = 'An unexpected error occurred. Please try again later.'; + +/** + * Stable, machine-readable error codes. Clients should branch on these rather + * than on HTTP status codes or message text. + */ +export const ERROR_CODES = { + BAD_REQUEST: 'BAD_REQUEST', + VALIDATION_FAILED: 'VALIDATION_FAILED', + UNAUTHORIZED: 'UNAUTHORIZED', + FORBIDDEN: 'FORBIDDEN', + NOT_FOUND: 'NOT_FOUND', + CONFLICT: 'CONFLICT', + UNPROCESSABLE_ENTITY: 'UNPROCESSABLE_ENTITY', + RATE_LIMITED: 'RATE_LIMITED', + INTERNAL_ERROR: 'INTERNAL_ERROR', + SERVICE_UNAVAILABLE: 'SERVICE_UNAVAILABLE', +} as const; + +export type ErrorCode = (typeof ERROR_CODES)[keyof typeof ERROR_CODES]; + +/** A single invalid field. Never contains the submitted value. */ +export interface ApiFieldError { + /** Dot-separated path, e.g. `payment.method`. */ + field: string; + /** Human-readable reason the field was rejected. */ + message: string; +} + +export interface ApiErrorBody { + version: string; + code: string; + message: string; + /** Correlation ID — matches the `X-Correlation-ID` response header. */ + requestId: string; + timestamp: string; + fieldErrors?: ApiFieldError[]; +} + +export interface ApiErrorEnvelope { + error: ApiErrorBody; +} + +/** Default code for a status code that was not raised through {@link ApiError}. */ +const CODE_BY_STATUS: Record = { + 400: ERROR_CODES.BAD_REQUEST, + 401: ERROR_CODES.UNAUTHORIZED, + 403: ERROR_CODES.FORBIDDEN, + 404: ERROR_CODES.NOT_FOUND, + 409: ERROR_CODES.CONFLICT, + 422: ERROR_CODES.UNPROCESSABLE_ENTITY, + 429: ERROR_CODES.RATE_LIMITED, + 503: ERROR_CODES.SERVICE_UNAVAILABLE, +}; + +export function defaultCodeForStatus(statusCode: number): ErrorCode { + return CODE_BY_STATUS[statusCode] ?? ERROR_CODES.INTERNAL_ERROR; +} + +/** + * Application error carrying everything the envelope needs. + * Throw this (directly or via the static helpers) from routes and services. + */ +export class ApiError extends Error { + readonly statusCode: number; + readonly code: ErrorCode | string; + readonly fieldErrors?: ApiFieldError[]; + /** + * Whether `message` is safe to return verbatim. Defaults to true for 4xx and + * false for 5xx, so internal details never reach clients by accident. + */ + readonly expose: boolean; + + constructor( + statusCode: number, + message: string, + options: { + code?: ErrorCode | string; + fieldErrors?: ApiFieldError[]; + expose?: boolean; + cause?: unknown; + } = {} + ) { + super(message); + this.name = 'ApiError'; + this.statusCode = statusCode; + this.code = options.code ?? defaultCodeForStatus(statusCode); + this.expose = options.expose ?? statusCode < 500; + if (options.fieldErrors) this.fieldErrors = options.fieldErrors; + if (options.cause !== undefined) this.cause = options.cause; + } + + static badRequest(message: string, fieldErrors?: ApiFieldError[]): ApiError { + return new ApiError(400, message, { + code: ERROR_CODES.BAD_REQUEST, + ...(fieldErrors && { fieldErrors }), + }); + } + + static validationFailed(message: string, fieldErrors: ApiFieldError[]): ApiError { + return new ApiError(400, message, { code: ERROR_CODES.VALIDATION_FAILED, fieldErrors }); + } + + static unauthorized(message = 'Authentication required'): ApiError { + return new ApiError(401, message, { code: ERROR_CODES.UNAUTHORIZED }); + } + + static forbidden(message = 'Insufficient permissions'): ApiError { + return new ApiError(403, message, { code: ERROR_CODES.FORBIDDEN }); + } + + static notFound(message = 'Resource not found'): ApiError { + return new ApiError(404, message, { code: ERROR_CODES.NOT_FOUND }); + } + + static conflict(message: string): ApiError { + return new ApiError(409, message, { code: ERROR_CODES.CONFLICT }); + } + + static rateLimited(message = 'Too many requests'): ApiError { + return new ApiError(429, message, { code: ERROR_CODES.RATE_LIMITED }); + } + + static internal(message = GENERIC_SERVER_MESSAGE, cause?: unknown): ApiError { + return new ApiError(500, message, { code: ERROR_CODES.INTERNAL_ERROR, expose: false, cause }); + } +} + +export function isApiError(error: unknown): error is ApiError { + return error instanceof ApiError; +} + +/** + * Resolve the correlation ID for a request. + * + * Prefers the ID assigned by the request logger, then inbound trace headers, + * and only generates one as a last resort so an error response is never + * missing an ID. + */ +export function getRequestId(req: Request): string { + const fromHeaders = + (req.headers['x-correlation-id'] as string | undefined) || + (req.headers['x-request-id'] as string | undefined); + + return req.correlationId || fromHeaders || randomUUID(); +} + +/** + * Build the envelope body. `message` is expected to already be client-safe. + */ +export function buildErrorEnvelope(input: { + code: ErrorCode | string; + message: string; + requestId: string; + fieldErrors?: ApiFieldError[]; + timestamp?: string; +}): ApiErrorEnvelope { + return { + error: { + version: ERROR_ENVELOPE_VERSION, + code: input.code, + message: input.message, + requestId: input.requestId, + timestamp: input.timestamp ?? new Date().toISOString(), + ...(input.fieldErrors && input.fieldErrors.length > 0 && { fieldErrors: input.fieldErrors }), + }, + }; +} + +/** + * Normalise any thrown value into { statusCode, envelope } and log the full + * detail server-side under the same request ID. + */ +export function toErrorResponse( + err: unknown, + requestId: string, + context: Record = {} +): { statusCode: number; envelope: ApiErrorEnvelope } { + const apiError = isApiError(err) ? err : null; + const statusCode = apiError?.statusCode ?? 500; + const code = apiError?.code ?? defaultCodeForStatus(statusCode); + + const rawMessage = err instanceof Error ? err.message : String(err); + const clientMessage = + apiError && apiError.expose + ? apiError.message + : statusCode < 500 + ? rawMessage + : GENERIC_SERVER_MESSAGE; + + // Detailed, correlated server-side record — stack traces stay here. + const logPayload = { + requestId, + code, + statusCode, + message: rawMessage, + stack: err instanceof Error ? err.stack : undefined, + ...context, + }; + + if (statusCode >= 500) { + logger.error(`Request failed [${code}]`, logPayload); + } else { + logger.warn(`Request rejected [${code}]`, logPayload); + } + + return { + statusCode, + envelope: buildErrorEnvelope({ + code, + message: clientMessage, + requestId, + ...(apiError?.fieldErrors && { fieldErrors: apiError.fieldErrors }), + }), + }; +} + +/** + * Send an envelope for `err`, always echoing the correlation ID as a header. + */ +export function sendErrorEnvelope(req: Request, res: Response, err: unknown): Response { + const requestId = getRequestId(req); + const { statusCode, envelope } = toErrorResponse(err, requestId, { + method: req.method, + path: req.originalUrl ?? req.url, + }); + + res.setHeader('X-Correlation-ID', requestId); + return res.status(statusCode).json(envelope); +} diff --git a/backend/tests/certificates.routes.test.ts b/backend/tests/certificates.routes.test.ts index 819a8097..0f6a788f 100644 --- a/backend/tests/certificates.routes.test.ts +++ b/backend/tests/certificates.routes.test.ts @@ -6,8 +6,10 @@ describe('Certificate route validation', () => { const response = await request(app).post('/api/v1/certificates').send({ courseId: 'course-101' }); expect(response.status).toBe(400); - expect(response.body.status).toBe('error'); - expect(response.body.errors).toEqual( + expect(response.body.error.code).toBe('VALIDATION_FAILED'); + expect(response.body.error.version).toBe('1'); + expect(response.body.error.requestId).toBeTruthy(); + expect(response.body.error.fieldErrors).toEqual( expect.arrayContaining([ expect.objectContaining({ field: 'studentId' }), ]) @@ -23,12 +25,15 @@ describe('Certificate route validation', () => { }); expect(response.status).toBe(400); - expect(response.body.status).toBe('error'); - expect(response.body.errors).toEqual( + expect(response.body.error.code).toBe('VALIDATION_FAILED'); + expect(response.body.error.fieldErrors).toEqual( expect.arrayContaining([ expect.objectContaining({ field: 'tokenId' }), expect.objectContaining({ field: 'did' }), ]) ); + // Validation failures must not leak internals. + expect(JSON.stringify(response.body)).not.toContain('at '); + expect(response.headers['x-correlation-id']).toBeTruthy(); }); }); diff --git a/backend/tests/errorEnvelope.routes.test.ts b/backend/tests/errorEnvelope.routes.test.ts new file mode 100644 index 00000000..b179a834 --- /dev/null +++ b/backend/tests/errorEnvelope.routes.test.ts @@ -0,0 +1,99 @@ +/** + * Route-level tests for the versioned API error envelope. + * + * A minimal Express app is assembled from the real middleware so the contract + * is exercised end-to-end without booting the whole server. + */ + +import express from 'express'; +import request from 'supertest'; +import { z } from 'zod'; +import { validate } from '../src/middleware/validation.js'; +import { errorHandler } from '../src/middleware/errorHandler.js'; +import { detailedRequestLogger } from '../src/middleware/requestLogger.js'; +import { ApiError } from '../src/utils/apiError.js'; + +const schema = z.object({ + email: z.string().email(), + age: z.number().int().positive(), +}); + +const buildApp = () => { + const app = express(); + app.use(express.json()); + app.use(detailedRequestLogger); + + app.post('/validated', validate(schema), (_req, res) => { + res.json({ ok: true }); + }); + + app.get('/not-found', (_req, _res, next) => { + next(ApiError.notFound('Certificate not found')); + }); + + app.get('/boom', (_req, _res, next) => { + next(new Error('database password is hunter2')); + }); + + app.use(errorHandler); + return app; +}; + +describe('API error envelope (routes)', () => { + const app = buildApp(); + + it('returns field errors for validation failures without leaking stack traces', async () => { + const res = await request(app).post('/validated').send({ email: 'nope', age: -1 }); + + expect(res.status).toBe(400); + expect(res.body.error.version).toBe('1'); + expect(res.body.error.code).toBe('VALIDATION_FAILED'); + expect(res.body.error.message).toBe('Request validation failed'); + expect(res.body.error.fieldErrors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ field: 'email' }), + expect.objectContaining({ field: 'age' }), + ]) + ); + + const serialized = JSON.stringify(res.body); + expect(serialized).not.toContain('ZodError'); + expect(serialized).not.toMatch(/\bat .*\.ts:\d+/); + }); + + it('includes a correlation id on every error response', async () => { + const res = await request(app).get('/not-found'); + + expect(res.status).toBe(404); + expect(res.body.error.code).toBe('NOT_FOUND'); + expect(res.body.error.requestId).toBeTruthy(); + expect(res.headers['x-correlation-id']).toBe(res.body.error.requestId); + }); + + it('echoes an inbound X-Correlation-ID so clients can trace a failure', async () => { + const res = await request(app) + .get('/not-found') + .set('X-Correlation-ID', 'trace-me-please'); + + expect(res.body.error.requestId).toBe('trace-me-please'); + }); + + it('hides internal failure detail behind a generic 500 message', async () => { + const res = await request(app).get('/boom'); + + expect(res.status).toBe(500); + expect(res.body.error.code).toBe('INTERNAL_ERROR'); + expect(res.body.error.message).toBe( + 'An unexpected error occurred. Please try again later.' + ); + expect(JSON.stringify(res.body)).not.toContain('hunter2'); + expect(res.body.error.requestId).toBeTruthy(); + }); + + it('passes valid requests through untouched', async () => { + const res = await request(app).post('/validated').send({ email: 'a@b.io', age: 21 }); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true }); + }); +}); diff --git a/backend/tests/sentry.errorHandler.test.ts b/backend/tests/sentry.errorHandler.test.ts index e451ca87..853872e2 100644 --- a/backend/tests/sentry.errorHandler.test.ts +++ b/backend/tests/sentry.errorHandler.test.ts @@ -1,27 +1,92 @@ import { errorHandler } from '../src/middleware/errorHandler.js'; import { captureException } from '../src/utils/sentry.js'; +import { ApiError, ERROR_ENVELOPE_VERSION } from '../src/utils/apiError.js'; jest.mock('../src/utils/sentry.js', () => ({ captureException: jest.fn(), })); -describe('Global Error Handler', () => { - it('captures exception and returns standardized 500 response', () => { +const mockRequest = (overrides: Record = {}) => + ({ + method: 'GET', + originalUrl: '/api/v1/example', + headers: {}, + ...overrides, + }) as any; + +const mockResponse = () => { + const res: any = { + headersSent: false, + status: jest.fn().mockReturnThis(), + json: jest.fn().mockReturnThis(), + setHeader: jest.fn().mockReturnThis(), + }; + return res; +}; + +describe('Global Error Handler (error envelope)', () => { + beforeEach(() => jest.clearAllMocks()); + + it('captures the exception and returns the versioned 500 envelope', () => { const err = new Error('test-error'); - const res: any = { - status: jest.fn().mockReturnThis(), - json: jest.fn(), - }; + const res = mockResponse(); const next = jest.fn(); - errorHandler(err, {} as any, res, next); + errorHandler(err, mockRequest({ correlationId: 'corr-123' }), res, next); expect(captureException).toHaveBeenCalledWith(err); expect(res.status).toHaveBeenCalledWith(500); - expect(res.json).toHaveBeenCalledWith({ - status: 'error', - message: 'Internal server error', - }); + expect(res.setHeader).toHaveBeenCalledWith('X-Correlation-ID', 'corr-123'); expect(next).not.toHaveBeenCalled(); + + const body = res.json.mock.calls[0][0]; + expect(body.error).toMatchObject({ + version: ERROR_ENVELOPE_VERSION, + code: 'INTERNAL_ERROR', + requestId: 'corr-123', + }); + // Internal detail must not reach the client. + expect(body.error.message).not.toContain('test-error'); + expect(JSON.stringify(body)).not.toContain('stack'); + }); + + it('preserves status, code and field errors from an ApiError', () => { + const err = ApiError.validationFailed('Request validation failed', [ + { field: 'tier', message: 'Invalid tier' }, + ]); + const res = mockResponse(); + + errorHandler(err, mockRequest({ correlationId: 'corr-456' }), res, jest.fn()); + + expect(res.status).toHaveBeenCalledWith(400); + const body = res.json.mock.calls[0][0]; + expect(body.error.code).toBe('VALIDATION_FAILED'); + expect(body.error.message).toBe('Request validation failed'); + expect(body.error.fieldErrors).toEqual([{ field: 'tier', message: 'Invalid tier' }]); + }); + + it('falls back to inbound trace headers for the request id', () => { + const res = mockResponse(); + + errorHandler( + ApiError.notFound('nope'), + mockRequest({ headers: { 'x-request-id': 'header-789' } }), + res, + jest.fn() + ); + + expect(res.json.mock.calls[0][0].error.requestId).toBe('header-789'); + }); + + it('delegates to next() once headers have been sent', () => { + const err = new Error('late failure'); + const res = mockResponse(); + res.headersSent = true; + const next = jest.fn(); + + errorHandler(err, mockRequest(), res, next); + + expect(next).toHaveBeenCalledWith(err); + expect(res.json).not.toHaveBeenCalled(); }); }); From 9f8eed14ea98df577a20be6b96dd1140239f6337 Mon Sep 17 00:00:00 2001 From: Abioladory123 Date: Thu, 30 Jul 2026 13:11:50 +0100 Subject: [PATCH 3/3] feat(observability): export cache and application metrics for Prometheus --- backend/.env.example | 10 + backend/METRICS_DOCUMENTATION.md | 131 ++++++++++ backend/src/cache/CacheMetrics.ts | 4 + backend/src/metrics/MetricsExporter.ts | 311 ++++++++++++++++++++++++ backend/src/metrics/WorkerRegistry.ts | 86 +++++++ backend/src/middleware/metricsAuth.ts | 73 ++++++ backend/src/routes/metrics.routes.ts | 112 ++++++++- backend/src/services/storage/worker.ts | 15 ++ backend/src/services/webhooks/worker.ts | 9 + backend/tests/metricsExporter.test.ts | 205 ++++++++++++++++ 10 files changed, 948 insertions(+), 8 deletions(-) create mode 100644 backend/METRICS_DOCUMENTATION.md create mode 100644 backend/src/metrics/MetricsExporter.ts create mode 100644 backend/src/metrics/WorkerRegistry.ts create mode 100644 backend/src/middleware/metricsAuth.ts create mode 100644 backend/tests/metricsExporter.test.ts diff --git a/backend/.env.example b/backend/.env.example index 9c87d0ac..030b13f7 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -196,3 +196,13 @@ BACKUP_S3_SECRET_ACCESS_KEY="" # Optional S3-compatible endpoint (e.g., MinIO, DigitalOcean Spaces) # Leave empty to use default AWS S3 endpoint BACKUP_S3_ENDPOINT="" + +# ============================================ +# Metrics / Monitoring (see METRICS_DOCUMENTATION.md) +# ============================================ +# Shared secret monitoring agents send as X-Metrics-Token or Authorization: Bearer. +# Required in production: without it the metrics endpoints return 503. +METRICS_AUTH_TOKEN="" + +# Max metrics scrapes per minute per identity +METRICS_RATE_LIMIT=120 diff --git a/backend/METRICS_DOCUMENTATION.md b/backend/METRICS_DOCUMENTATION.md new file mode 100644 index 00000000..d170b2da --- /dev/null +++ b/backend/METRICS_DOCUMENTATION.md @@ -0,0 +1,131 @@ +# Metrics & Monitoring + +The backend exports aggregated in-process metrics in a form monitoring systems +can consume without manual parsing. + +## Endpoints + +| Endpoint | Format | Purpose | +| --- | --- | --- | +| `GET /api/v1/metrics/prometheus` | `text/plain; version=0.0.4` | Scrape target. Stable names, units in the name. | +| `GET /api/v1/metrics/snapshot` | JSON (`schemaVersion: "1"`) | Same aggregation for JSON-only tooling. | +| `GET /api/v1/metrics` | JSON | Legacy summary shape. | +| `GET /api/v1/metrics/performance` | JSON | Retained per-request samples (ring buffer). | +| `GET /api/v1/metrics/errors` | JSON | Error entries with **messages redacted**. | +| `GET /api/v1/metrics/business` | JSON | Domain event entries. | +| `POST /api/v1/metrics/reset` | JSON | Clears counters (admin/manual use). | +| `GET /api/v1/cache/metrics` | JSON | Cache hit/miss plus backend reachability. | + +### Authorization and rate control + +All of the above require the monitoring secret `METRICS_AUTH_TOKEN`, sent as +either header: + +``` +X-Metrics-Token: +Authorization: Bearer +``` + +Comparison is constant-time. If `METRICS_AUTH_TOKEN` is unset the endpoints stay +open in development and test but return `503 SERVICE_UNAVAILABLE` in production, +so a deployed instance is never unprotected. Scrapes are rate limited to +`METRICS_RATE_LIMIT` requests/minute per identity (default 120). Failures use the +standard [error envelope](./API_ERROR_CONTRACT.md). + +Example Prometheus scrape config: + +```yaml +scrape_configs: + - job_name: web3-student-lab-api + scrape_interval: 30s + metrics_path: /api/v1/metrics/prometheus + static_configs: + - targets: ['api.internal:8080'] + authorization: + type: Bearer + credentials_file: /etc/prometheus/w3sl-metrics-token +``` + +## Exported metrics + +All names are prefixed `w3sl_`. Counters are monotonic per process lifetime. + +| Metric | Type | Unit | Description | +| --- | --- | --- | --- | +| `w3sl_cache_backend_up{mode}` | gauge | boolean | 1 = cache backend reachable, 0 = unreachable. `mode` is `standalone`/`cluster`/`sentinel`. | +| `w3sl_cache_hits_total` | counter | lookups | Lookups served from cache. | +| `w3sl_cache_misses_total` | counter | lookups | Lookups that missed. | +| `w3sl_cache_hit_ratio` | gauge | ratio 0–1 | Hit ratio over the process lifetime. | +| `w3sl_http_requests_total{method,route}` | counter | requests | Requests by method and **normalised** route. | +| `w3sl_http_responses_total{status_class}` | counter | responses | Responses by `2xx`/`4xx`/`5xx`. | +| `w3sl_http_request_duration_milliseconds_avg` | gauge | ms | Mean duration over retained samples. | +| `w3sl_errors_total{type}` | counter | errors | Errors by type/class name. `type="all"` is the total. | +| `w3sl_business_events_total{event}` | counter | events | Domain events, e.g. `certificate.minted`. | +| `w3sl_worker_up{worker,state}` | gauge | boolean | 1 = running, 0 = stopped/degraded. | +| `w3sl_worker_jobs_completed_total{worker}` | counter | jobs | Jobs completed per worker. | +| `w3sl_worker_jobs_failed_total{worker}` | counter | jobs | Jobs failed per worker. | +| `w3sl_process_uptime_seconds` | gauge | seconds | Process uptime. | +| `w3sl_process_resident_memory_bytes` | gauge | bytes | Node heap usage. | +| `w3sl_process_cpu_user_seconds_total` | counter | seconds | User CPU time. | + +Known `worker` labels: `storage-pin`, `storage-gc`, `webhook-delivery`. + +### What is deliberately excluded + +- Request and response bodies, query strings and headers. +- User, student and wallet identifiers — route labels have identifier-looking + segments rewritten to `:id` (`/certificates/4242` → `/certificates/:id`), which + also keeps label cardinality bounded. +- Error *messages*. Only the error type is exported; the full message and stack + live in the logs, correlated by the `requestId` from the error envelope. +- Business event metadata (only the event name and count are exported). + +## Dashboards + +**API health** +1. Request rate — `sum(rate(w3sl_http_requests_total[5m]))` +2. Error ratio — `sum(rate(w3sl_http_responses_total{status_class="5xx"}[5m])) / sum(rate(w3sl_http_responses_total[5m]))` +3. Mean latency — `w3sl_http_request_duration_milliseconds_avg` +4. Top routes — `topk(10, sum by (route) (rate(w3sl_http_requests_total[5m])))` + +**Cache health** +1. `w3sl_cache_backend_up` as a status tile +2. `w3sl_cache_hit_ratio` trend +3. Lookup rate — `rate(w3sl_cache_hits_total[5m])` vs `rate(w3sl_cache_misses_total[5m])` + +**Workers** +1. `w3sl_worker_up` per worker as status tiles +2. Failure rate — `rate(w3sl_worker_jobs_failed_total[15m])` +3. Throughput — `rate(w3sl_worker_jobs_completed_total[15m])` + +**Process**: uptime (restart detection), resident memory, CPU seconds. + +## Alert-worthy signals + +| Alert | Condition | Severity | +| --- | --- | --- | +| Cache backend down | `w3sl_cache_backend_up == 0` for 2m | critical | +| Elevated 5xx | 5xx ratio > 2% for 5m | critical | +| Latency regression | `w3sl_http_request_duration_milliseconds_avg > 1000` for 10m | warning | +| Cache hit ratio collapse | `w3sl_cache_hit_ratio < 0.5` for 15m (with non-trivial lookup rate) | warning | +| Worker down | `w3sl_worker_up == 0` for 5m while the app is up | critical | +| Worker failures | `rate(w3sl_worker_jobs_failed_total[15m]) > 0.1` | warning | +| Memory growth | `w3sl_process_resident_memory_bytes` up >50% over 1h with flat traffic | warning | +| Frequent restarts | `w3sl_process_uptime_seconds` resets more than twice in 30m | warning | +| Scrape failure | target down for 5m | warning | + +When an alert fires, take the correlation ID from the affected request's error +envelope (or the log entry) and search the logs — metrics intentionally carry no +request detail. + +## Implementation notes + +Counters are per-process and in memory: they reset on restart, and with multiple +instances each one must be scraped separately (aggregate in the monitoring +system). Retained raw samples are bounded to 10,000 entries per category +(ring buffer) in `src/metrics/MetricsCollector.ts`. + +- `src/metrics/MetricsExporter.ts` — snapshot + Prometheus rendering +- `src/metrics/WorkerRegistry.ts` — worker liveness and job counters +- `src/middleware/metricsAuth.ts` — monitoring token check +- `tests/metricsExporter.test.ts` — schema, label safety and auth tests diff --git a/backend/src/cache/CacheMetrics.ts b/backend/src/cache/CacheMetrics.ts index 4af6acee..8ef8e48e 100644 --- a/backend/src/cache/CacheMetrics.ts +++ b/backend/src/cache/CacheMetrics.ts @@ -1,9 +1,13 @@ import { Router } from 'express'; import cacheService from './CacheService.js'; import redisClient from './RedisClient.js'; +import { requireMetricsAuth } from '../middleware/metricsAuth.js'; const router = Router(); +// Cache metrics are operational data — same authorization as /api/v1/metrics. +router.use(requireMetricsAuth); + router.get('/metrics', (_req, res) => { const metrics = cacheService.getMetrics(); const isHealthy = redisClient.isHealthy(); diff --git a/backend/src/metrics/MetricsExporter.ts b/backend/src/metrics/MetricsExporter.ts new file mode 100644 index 00000000..12f93dbb --- /dev/null +++ b/backend/src/metrics/MetricsExporter.ts @@ -0,0 +1,311 @@ +/** + * MetricsExporter — turns the in-memory metrics into a production-consumable + * form: a stable JSON snapshot and a Prometheus text-exposition rendering. + * + * Design rules: + * - Stable names, explicit units in the name (`_seconds`, `_bytes`, `_total`). + * - Aggregates only. No request bodies, no user identifiers, no error + * messages — error metrics are keyed by error *type* alone. + * - Route labels are normalised (`/certificates/abc123` → `/certificates/:id`) + * so identifiers never reach the monitoring system and label cardinality + * stays bounded. + */ + +import metricsCollector, { type MetricsSummary } from './MetricsCollector.js'; +import workerRegistry, { type WorkerStatus } from './WorkerRegistry.js'; +import cacheService from '../cache/CacheService.js'; +import redisClient from '../cache/RedisClient.js'; + +/** Snapshot schema version. Bump on a breaking field change. */ +export const METRICS_SCHEMA_VERSION = '1'; + +/** Metric name prefix for everything this exporter emits. */ +const PREFIX = 'w3sl'; + +export interface CacheMetricsSnapshot { + /** 1 when the cache backend is reachable, 0 otherwise. */ + backendUp: 0 | 1; + backendMode: string; + hitsTotal: number; + missesTotal: number; + /** Hit ratio in the range 0–1 (not a percentage). */ + hitRatio: number; +} + +export interface HttpMetricsSnapshot { + requestsTotal: number; + averageDurationMilliseconds: number; + /** Requests keyed by normalised `METHOD /route`. */ + requestsByRoute: Record; + /** Requests keyed by status class: `2xx`, `4xx`, `5xx`. */ + requestsByStatusClass: Record; +} + +export interface ErrorMetricsSnapshot { + errorsTotal: number; + /** Counts keyed by error type/class name only — never by message. */ + errorsByType: Record; +} + +export interface MetricsSnapshot { + schemaVersion: string; + collectedAt: string; + cache: CacheMetricsSnapshot; + http: HttpMetricsSnapshot; + errors: ErrorMetricsSnapshot; + business: { eventsTotal: number; eventsByName: Record }; + workers: WorkerStatus[]; + system: { uptimeSeconds: number; memoryResidentBytes: number; cpuUserSeconds: number }; +} + +/** Segments that look like identifiers rather than route names. */ +const ID_SEGMENT = /^(?:[0-9]+|c[a-z0-9]{20,}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-fA-F]{16,}|G[A-Z2-7]{55})$/; + +/** + * Replace identifier-looking path segments with `:id` so metric labels never + * carry certificate IDs, wallet addresses or user IDs. + */ +export function normalizeRouteLabel(route: string): string { + const [pathOnly = ''] = route.split('?'); + return ( + pathOnly + .split('/') + .map((segment) => (segment && ID_SEGMENT.test(segment) ? ':id' : segment)) + .join('/') || '/' + ); +} + +/** Bucket a status code into a low-cardinality class label. */ +function statusClass(statusCode: string): string { + const first = statusCode.charAt(0); + return /[1-5]/.test(first) ? `${first}xx` : 'unknown'; +} + +function parseHitRatio(hitRate: string): number { + const value = Number.parseFloat(hitRate.replace('%', '')); + return Number.isFinite(value) ? Number((value / 100).toFixed(4)) : 0; +} + +function aggregateRoutes(requestsByRoute: Record): Record { + const out: Record = {}; + for (const [key, count] of Object.entries(requestsByRoute)) { + const spaceIdx = key.indexOf(' '); + const method = spaceIdx === -1 ? '' : key.slice(0, spaceIdx); + const route = spaceIdx === -1 ? key : key.slice(spaceIdx + 1); + const normalized = method ? `${method} ${normalizeRouteLabel(route)}` : normalizeRouteLabel(route); + out[normalized] = (out[normalized] ?? 0) + count; + } + return out; +} + +function aggregateStatusClasses(requestsByStatus: Record): Record { + const out: Record = {}; + for (const [status, count] of Object.entries(requestsByStatus)) { + const cls = statusClass(status); + out[cls] = (out[cls] ?? 0) + count; + } + return out; +} + +/** + * Build the stable snapshot. Safe to call on every scrape — it only reads + * already-aggregated in-process counters. + */ +export function buildMetricsSnapshot( + summary: MetricsSummary = metricsCollector.getSummary() +): MetricsSnapshot { + const cacheMetrics = cacheService.getMetrics(); + const backendUp: 0 | 1 = redisClient.isHealthy() ? 1 : 0; + + return { + schemaVersion: METRICS_SCHEMA_VERSION, + collectedAt: summary.collectedAt, + cache: { + backendUp, + backendMode: redisClient.getMode(), + hitsTotal: cacheMetrics.hits, + missesTotal: cacheMetrics.misses, + hitRatio: parseHitRatio(cacheMetrics.hitRate), + }, + http: { + requestsTotal: summary.performance.totalRequests, + averageDurationMilliseconds: summary.performance.averageDurationMs, + requestsByRoute: aggregateRoutes(summary.performance.requestsByRoute), + requestsByStatusClass: aggregateStatusClasses(summary.performance.requestsByStatus), + }, + errors: { + errorsTotal: summary.errors.totalErrors, + errorsByType: summary.errors.errorsByType, + }, + business: { + eventsTotal: summary.business.totalEvents, + eventsByName: summary.business.eventsByName, + }, + workers: workerRegistry.list(), + system: { + uptimeSeconds: summary.system.uptimeSeconds, + memoryResidentBytes: Math.round(summary.system.memoryUsageMB * 1024 * 1024), + cpuUserSeconds: Number((summary.system.cpuUserMs / 1000).toFixed(3)), + }, + }; +} + +// ─── Prometheus text exposition ─────────────────────────────────────────────── + +/** Escape a label value per the Prometheus exposition format. */ +function escapeLabelValue(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, ' '); +} + +/** Keep label values bounded and free of anything payload-like. */ +function safeLabelValue(value: string): string { + return escapeLabelValue(value.slice(0, 120)); +} + +function renderLabels(labels: Record): string { + const entries = Object.entries(labels); + if (entries.length === 0) return ''; + return `{${entries.map(([k, v]) => `${k}="${safeLabelValue(v)}"`).join(',')}}`; +} + +interface MetricFamily { + name: string; + help: string; + type: 'counter' | 'gauge'; + samples: Array<{ labels?: Record; value: number }>; +} + +function renderFamily(family: MetricFamily): string { + const lines = [ + `# HELP ${family.name} ${family.help}`, + `# TYPE ${family.name} ${family.type}`, + ]; + for (const sample of family.samples) { + lines.push(`${family.name}${renderLabels(sample.labels ?? {})} ${sample.value}`); + } + return lines.join('\n'); +} + +/** + * Render a snapshot as Prometheus text exposition (content type + * `text/plain; version=0.0.4`). Names, units and help strings are stable. + */ +export function renderPrometheus(snapshot: MetricsSnapshot = buildMetricsSnapshot()): string { + const families: MetricFamily[] = [ + { + name: `${PREFIX}_cache_backend_up`, + help: 'Cache backend reachability: 1 = reachable, 0 = unreachable.', + type: 'gauge', + samples: [{ labels: { mode: snapshot.cache.backendMode }, value: snapshot.cache.backendUp }], + }, + { + name: `${PREFIX}_cache_hits_total`, + help: 'Total cache lookups served from cache since process start.', + type: 'counter', + samples: [{ value: snapshot.cache.hitsTotal }], + }, + { + name: `${PREFIX}_cache_misses_total`, + help: 'Total cache lookups that missed since process start.', + type: 'counter', + samples: [{ value: snapshot.cache.missesTotal }], + }, + { + name: `${PREFIX}_cache_hit_ratio`, + help: 'Cache hit ratio over the process lifetime, 0-1.', + type: 'gauge', + samples: [{ value: snapshot.cache.hitRatio }], + }, + { + name: `${PREFIX}_http_requests_total`, + help: 'HTTP requests handled, by method and normalised route.', + type: 'counter', + samples: Object.entries(snapshot.http.requestsByRoute).map(([key, value]) => { + const spaceIdx = key.indexOf(' '); + const method = spaceIdx === -1 ? 'UNKNOWN' : key.slice(0, spaceIdx); + const route = spaceIdx === -1 ? key : key.slice(spaceIdx + 1); + return { labels: { method, route }, value }; + }), + }, + { + name: `${PREFIX}_http_responses_total`, + help: 'HTTP responses handled, by status class (2xx/4xx/5xx).', + type: 'counter', + samples: Object.entries(snapshot.http.requestsByStatusClass).map(([cls, value]) => ({ + labels: { status_class: cls }, + value, + })), + }, + { + name: `${PREFIX}_http_request_duration_milliseconds_avg`, + help: 'Mean HTTP request duration in milliseconds over retained samples.', + type: 'gauge', + samples: [{ value: snapshot.http.averageDurationMilliseconds }], + }, + { + name: `${PREFIX}_errors_total`, + help: 'Application errors recorded, by error type. Messages are not exported.', + type: 'counter', + samples: [ + { labels: { type: 'all' }, value: snapshot.errors.errorsTotal }, + ...Object.entries(snapshot.errors.errorsByType).map(([type, value]) => ({ + labels: { type }, + value, + })), + ], + }, + { + name: `${PREFIX}_business_events_total`, + help: 'Domain events recorded, by event name.', + type: 'counter', + samples: Object.entries(snapshot.business.eventsByName).map(([event, value]) => ({ + labels: { event }, + value, + })), + }, + { + name: `${PREFIX}_worker_up`, + help: 'Background worker state: 1 = running, 0 = stopped or degraded.', + type: 'gauge', + samples: snapshot.workers.map((w) => ({ + labels: { worker: w.name, state: w.state }, + value: w.state === 'running' ? 1 : 0, + })), + }, + { + name: `${PREFIX}_worker_jobs_completed_total`, + help: 'Background jobs completed successfully, by worker.', + type: 'counter', + samples: snapshot.workers.map((w) => ({ labels: { worker: w.name }, value: w.jobsCompleted })), + }, + { + name: `${PREFIX}_worker_jobs_failed_total`, + help: 'Background jobs that failed, by worker.', + type: 'counter', + samples: snapshot.workers.map((w) => ({ labels: { worker: w.name }, value: w.jobsFailed })), + }, + { + name: `${PREFIX}_process_uptime_seconds`, + help: 'Process uptime in seconds.', + type: 'gauge', + samples: [{ value: snapshot.system.uptimeSeconds }], + }, + { + name: `${PREFIX}_process_resident_memory_bytes`, + help: 'Resident heap usage of the Node.js process in bytes.', + type: 'gauge', + samples: [{ value: snapshot.system.memoryResidentBytes }], + }, + { + name: `${PREFIX}_process_cpu_user_seconds_total`, + help: 'User CPU time consumed by the process in seconds.', + type: 'counter', + samples: [{ value: snapshot.system.cpuUserSeconds }], + }, + ]; + + return `${families.map(renderFamily).join('\n')}\n`; +} + +/** Content type expected by Prometheus-compatible scrapers. */ +export const PROMETHEUS_CONTENT_TYPE = 'text/plain; version=0.0.4; charset=utf-8'; diff --git a/backend/src/metrics/WorkerRegistry.ts b/backend/src/metrics/WorkerRegistry.ts new file mode 100644 index 00000000..43d6b981 --- /dev/null +++ b/backend/src/metrics/WorkerRegistry.ts @@ -0,0 +1,86 @@ +/** + * WorkerRegistry — tracks background worker liveness for the metrics exporter. + * + * Workers register themselves on start, deregister on stop, and report job + * outcomes. Only aggregate counters and state are stored: no job payloads, + * user identifiers or error messages, so the data is safe to export. + */ + +export type WorkerState = 'running' | 'stopped' | 'degraded'; + +export interface WorkerStatus { + /** Stable worker name used as the `worker` metric label. */ + name: string; + state: WorkerState; + /** Configured concurrency, when the worker exposes one. */ + concurrency?: number; + jobsCompleted: number; + jobsFailed: number; + /** ISO timestamp of the last state change or job outcome. */ + lastUpdatedAt: string; +} + +class WorkerRegistry { + private workers = new Map(); + + private upsert(name: string, patch: Partial): WorkerStatus { + const existing: WorkerStatus = this.workers.get(name) ?? { + name, + state: 'stopped', + jobsCompleted: 0, + jobsFailed: 0, + lastUpdatedAt: new Date().toISOString(), + }; + + const updated: WorkerStatus = { + ...existing, + ...patch, + name, + lastUpdatedAt: new Date().toISOString(), + }; + + this.workers.set(name, updated); + return updated; + } + + /** Mark a worker as running. Idempotent — safe to call on every start. */ + register(name: string, options: { concurrency?: number } = {}): void { + this.upsert(name, { + state: 'running', + ...(options.concurrency !== undefined && { concurrency: options.concurrency }), + }); + } + + /** Mark a worker as stopped while retaining its counters. */ + markStopped(name: string): void { + this.upsert(name, { state: 'stopped' }); + } + + /** Mark a worker as running but unhealthy (e.g. repeated connection errors). */ + markDegraded(name: string): void { + this.upsert(name, { state: 'degraded' }); + } + + recordCompleted(name: string): void { + const current = this.workers.get(name); + this.upsert(name, { jobsCompleted: (current?.jobsCompleted ?? 0) + 1 }); + } + + recordFailed(name: string): void { + const current = this.workers.get(name); + this.upsert(name, { jobsFailed: (current?.jobsFailed ?? 0) + 1 }); + } + + /** Snapshot of every known worker, ordered by name for stable metric output. */ + list(): WorkerStatus[] { + return [...this.workers.values()].sort((a, b) => a.name.localeCompare(b.name)); + } + + /** Test helper — drops all registrations. */ + reset(): void { + this.workers.clear(); + } +} + +export const workerRegistry = new WorkerRegistry(); +export default workerRegistry; diff --git a/backend/src/middleware/metricsAuth.ts b/backend/src/middleware/metricsAuth.ts new file mode 100644 index 00000000..3515bb14 --- /dev/null +++ b/backend/src/middleware/metricsAuth.ts @@ -0,0 +1,73 @@ +/** + * Authorization for operational metrics endpoints. + * + * Monitoring agents authenticate with a shared secret, supplied either as + * `X-Metrics-Token: ` or `Authorization: Bearer `, compared in + * constant time against `METRICS_AUTH_TOKEN`. + * + * If the variable is unset the endpoints stay open in development and test + * (so the local dashboards and the test suite keep working) but are refused in + * production — metrics are never silently unprotected in a deployed + * environment. + */ + +import { Request, Response, NextFunction } from 'express'; +import { timingSafeEqual } from 'node:crypto'; +import logger from '../utils/logger.js'; +import { ApiError, sendErrorEnvelope } from '../utils/apiError.js'; + +function constantTimeEquals(a: string, b: string): boolean { + const left = Buffer.from(a, 'utf8'); + const right = Buffer.from(b, 'utf8'); + if (left.length !== right.length) return false; + return timingSafeEqual(left, right); +} + +function extractToken(req: Request): string | undefined { + const headerToken = req.headers['x-metrics-token']; + if (typeof headerToken === 'string' && headerToken.length > 0) { + return headerToken; + } + + const authorization = req.headers.authorization; + if (typeof authorization === 'string' && authorization.startsWith('Bearer ')) { + return authorization.slice('Bearer '.length); + } + + return undefined; +} + +/** Warn once per process rather than on every scrape. */ +let warnedAboutMissingToken = false; + +export const requireMetricsAuth = (req: Request, res: Response, next: NextFunction) => { + const expected = process.env.METRICS_AUTH_TOKEN; + + if (!expected) { + if (process.env.NODE_ENV === 'production') { + return sendErrorEnvelope( + req, + res, + new ApiError(503, 'Metrics endpoint is not configured', { + code: 'SERVICE_UNAVAILABLE', + expose: true, + }) + ); + } + + if (!warnedAboutMissingToken) { + warnedAboutMissingToken = true; + logger.warn('METRICS_AUTH_TOKEN is not set — metrics endpoints are unauthenticated'); + } + return next(); + } + + const provided = extractToken(req); + if (!provided || !constantTimeEquals(provided, expected)) { + return sendErrorEnvelope(req, res, ApiError.unauthorized('Invalid or missing metrics token')); + } + + return next(); +}; + +export default requireMetricsAuth; diff --git a/backend/src/routes/metrics.routes.ts b/backend/src/routes/metrics.routes.ts index 1666f299..9bce3515 100644 --- a/backend/src/routes/metrics.routes.ts +++ b/backend/src/routes/metrics.routes.ts @@ -1,32 +1,111 @@ /** - * Metrics Routes — exposes collected metrics over HTTP. + * Metrics Routes — exposes collected metrics to operational tooling. * * Endpoints: - * GET /api/v1/metrics — aggregated summary + * GET /api/v1/metrics/prometheus — Prometheus text exposition (scrape here) + * GET /api/v1/metrics/snapshot — same data as stable JSON + * GET /api/v1/metrics — aggregated summary (legacy shape) * GET /api/v1/metrics/performance — raw performance entries - * GET /api/v1/metrics/errors — raw error entries + * GET /api/v1/metrics/errors — error entries, messages redacted * GET /api/v1/metrics/business — raw business event entries * POST /api/v1/metrics/reset — clear all metrics (admin use) * - * Educational note: In a real deployment you would protect these endpoints - * with an admin-only auth middleware. Here we keep it simple and rely on - * the existing workspace/rate-limit middleware applied at the router level. + * The whole router is behind `requireMetricsAuth` (shared monitoring token) and + * a sliding-window rate limit, consistent with the other operational + * endpoints. See METRICS_DOCUMENTATION.md for metric names, units, dashboards + * and alert thresholds. */ import { Router, Request, Response } from 'express'; import metricsCollector from '../metrics/MetricsCollector.js'; +import { + METRICS_SCHEMA_VERSION, + PROMETHEUS_CONTENT_TYPE, + buildMetricsSnapshot, + renderPrometheus, +} from '../metrics/MetricsExporter.js'; +import { requireMetricsAuth } from '../middleware/metricsAuth.js'; +import { slidingWindowRateLimiter } from '../middleware/rateLimiter.js'; const router = Router(); +// Operational access controls: shared-secret auth + scrape rate ceiling. +router.use(requireMetricsAuth); +router.use( + slidingWindowRateLimiter({ + windowMs: 60_000, + limit: Number(process.env.METRICS_RATE_LIMIT || '120'), + keyPrefix: 'rl:metrics', + }) +); + +/** + * @openapi + * /api/v1/metrics/prometheus: + * get: + * summary: Scrape metrics in Prometheus text exposition format + * description: > + * Stable metric names with units in the name. Aggregates only — no request + * bodies, user identifiers or error messages. Route labels are normalised + * so resource identifiers are never exported. + * tags: [Metrics] + * security: + * - metricsToken: [] + * responses: + * 200: + * description: Prometheus exposition payload + * content: + * text/plain: + * schema: + * type: string + * 401: + * $ref: '#/components/responses/Unauthorized' + * 429: + * $ref: '#/components/responses/RateLimited' + */ +router.get('/prometheus', (_req: Request, res: Response) => { + res.setHeader('Content-Type', PROMETHEUS_CONTENT_TYPE); + res.status(200).send(renderPrometheus()); +}); + +/** + * @openapi + * /api/v1/metrics/snapshot: + * get: + * summary: Get the stable JSON metrics snapshot + * description: > + * Same aggregation as the Prometheus endpoint, for tooling that prefers + * JSON. `schemaVersion` changes only on a breaking field change. + * tags: [Metrics] + * security: + * - metricsToken: [] + * responses: + * 200: + * description: Metrics snapshot + * 401: + * $ref: '#/components/responses/Unauthorized' + */ +router.get('/snapshot', (_req: Request, res: Response) => { + res.json({ + status: 'success', + schemaVersion: METRICS_SCHEMA_VERSION, + data: buildMetricsSnapshot(), + }); +}); + /** * @openapi * /api/v1/metrics: * get: * summary: Get aggregated metrics summary * tags: [Metrics] + * security: + * - metricsToken: [] * responses: * 200: * description: Metrics summary + * 401: + * $ref: '#/components/responses/Unauthorized' */ router.get('/', (_req: Request, res: Response) => { res.json({ status: 'success', data: metricsCollector.getSummary() }); @@ -38,6 +117,8 @@ router.get('/', (_req: Request, res: Response) => { * get: * summary: Get raw performance metrics * tags: [Metrics] + * security: + * - metricsToken: [] */ router.get('/performance', (_req: Request, res: Response) => { res.json({ status: 'success', data: metricsCollector.getPerformanceMetrics() }); @@ -47,11 +128,22 @@ router.get('/performance', (_req: Request, res: Response) => { * @openapi * /api/v1/metrics/errors: * get: - * summary: Get raw error metrics + * summary: Get error metrics with messages redacted + * description: > + * Error messages can contain request or user detail, so only the error + * type, status code and timestamp are returned. Use the correlation ID in + * the server logs for full detail. * tags: [Metrics] + * security: + * - metricsToken: [] */ router.get('/errors', (_req: Request, res: Response) => { - res.json({ status: 'success', data: metricsCollector.getErrorMetrics() }); + const redacted = metricsCollector.getErrorMetrics().map((entry) => ({ + type: entry.type, + statusCode: entry.statusCode, + timestamp: entry.timestamp, + })); + res.json({ status: 'success', data: redacted }); }); /** @@ -60,6 +152,8 @@ router.get('/errors', (_req: Request, res: Response) => { * get: * summary: Get raw business event metrics * tags: [Metrics] + * security: + * - metricsToken: [] */ router.get('/business', (_req: Request, res: Response) => { res.json({ status: 'success', data: metricsCollector.getBusinessMetrics() }); @@ -71,6 +165,8 @@ router.get('/business', (_req: Request, res: Response) => { * post: * summary: Reset all collected metrics * tags: [Metrics] + * security: + * - metricsToken: [] */ router.post('/reset', (_req: Request, res: Response) => { metricsCollector.reset(); diff --git a/backend/src/services/storage/worker.ts b/backend/src/services/storage/worker.ts index d8887b24..c72aa50c 100644 --- a/backend/src/services/storage/worker.ts +++ b/backend/src/services/storage/worker.ts @@ -4,6 +4,7 @@ import logger from '../../utils/logger.js'; import * as defaultRepository from './asset.repository.js'; import { createStorageProvider } from './provider.js'; import { STORAGE_GC_QUEUE_NAME, STORAGE_PIN_QUEUE_NAME, storageGcQueue } from './queue.js'; +import workerRegistry from '../../metrics/WorkerRegistry.js'; import type { StorageAssetRecord, StorageGcJobData, @@ -145,7 +146,14 @@ export const startStorageWorkers = (): { concurrency: Number(process.env.STORAGE_WORKER_CONCURRENCY || '10'), }); + workerRegistry.register('storage-pin', { + concurrency: Number(process.env.STORAGE_WORKER_CONCURRENCY || '10'), + }); + + pinWorker.on('completed', () => workerRegistry.recordCompleted('storage-pin')); + pinWorker.on('error', () => workerRegistry.markDegraded('storage-pin')); pinWorker.on('failed', (job, error) => { + workerRegistry.recordFailed('storage-pin'); logger.error(`Storage pin job ${job?.id} failed: ${error.message}`); }); } @@ -171,7 +179,12 @@ export const startStorageWorkers = (): { } ); + workerRegistry.register('storage-gc', { concurrency: 1 }); + + gcWorker.on('completed', () => workerRegistry.recordCompleted('storage-gc')); + gcWorker.on('error', () => workerRegistry.markDegraded('storage-gc')); gcWorker.on('failed', (job, error) => { + workerRegistry.recordFailed('storage-gc'); logger.error(`Storage GC job ${job?.id} failed: ${error.message}`); }); } @@ -183,11 +196,13 @@ export const stopStorageWorkers = async (): Promise => { if (pinWorker) { await pinWorker.close(); pinWorker = null; + workerRegistry.markStopped('storage-pin'); } if (gcWorker) { await gcWorker.close(); gcWorker = null; + workerRegistry.markStopped('storage-gc'); } }; diff --git a/backend/src/services/webhooks/worker.ts b/backend/src/services/webhooks/worker.ts index 6cd91784..99f291d8 100644 --- a/backend/src/services/webhooks/worker.ts +++ b/backend/src/services/webhooks/worker.ts @@ -7,6 +7,7 @@ import { import { buildSignedWebhookHeaders, canonicalizeWebhookPayload } from './signature.js'; import type { DeadLetterWebhookJob, WebhookDeliveryJobData } from './types.js'; import { recordDeliveryState } from './dispatcher.js'; +import workerRegistry from '../../metrics/WorkerRegistry.js'; const requestTimeoutMs = Number(process.env.WEBHOOK_REQUEST_TIMEOUT_MS || '10000'); @@ -181,6 +182,8 @@ export const startWebhookWorker = (): Worker | null => { ); webhookWorker.on('failed', async (job, error) => { + workerRegistry.recordFailed('webhook-delivery'); + if (!job) { return; } @@ -211,13 +214,18 @@ export const startWebhookWorker = (): Worker | null => { }); webhookWorker.on('completed', (job) => { + workerRegistry.recordCompleted('webhook-delivery'); logger.info(`Webhook delivery ${job.data.deliveryId} completed`); }); webhookWorker.on('error', (error) => { + workerRegistry.markDegraded('webhook-delivery'); logger.error('Webhook worker error:', error); }); + // Visible to the metrics exporter as worker="webhook-delivery". + workerRegistry.register('webhook-delivery'); + return webhookWorker; }; @@ -228,4 +236,5 @@ export const stopWebhookWorker = async (): Promise => { await webhookWorker.close(); webhookWorker = null; + workerRegistry.markStopped('webhook-delivery'); }; diff --git a/backend/tests/metricsExporter.test.ts b/backend/tests/metricsExporter.test.ts new file mode 100644 index 00000000..8c0fe73e --- /dev/null +++ b/backend/tests/metricsExporter.test.ts @@ -0,0 +1,205 @@ +/** + * Tests for the production metrics export: stable schema, safe labels and the + * authorization applied to the operational endpoints. + */ + +import express from 'express'; +import request from 'supertest'; +import metricsCollector from '../src/metrics/MetricsCollector.js'; +import workerRegistry from '../src/metrics/WorkerRegistry.js'; +import { + METRICS_SCHEMA_VERSION, + buildMetricsSnapshot, + normalizeRouteLabel, + renderPrometheus, +} from '../src/metrics/MetricsExporter.js'; +import metricsRouter from '../src/routes/metrics.routes.js'; +import { requireMetricsAuth } from '../src/middleware/metricsAuth.js'; + +describe('MetricsExporter.normalizeRouteLabel', () => { + it('replaces identifier-looking segments so labels stay bounded', () => { + expect(normalizeRouteLabel('/api/v1/certificates/4242/metadata')).toBe( + '/api/v1/certificates/:id/metadata' + ); + expect( + normalizeRouteLabel('/api/v1/students/ckq1x8z9a0000abcdefghijkl/certificates') + ).toBe('/api/v1/students/:id/certificates'); + expect( + normalizeRouteLabel('/api/v1/users/9f1c2e3a-6b74-4c0f-9a5c-7b1d2e3f4a5b') + ).toBe('/api/v1/users/:id'); + }); + + it('leaves plain routes and query strings alone', () => { + expect(normalizeRouteLabel('/api/v1/courses')).toBe('/api/v1/courses'); + expect(normalizeRouteLabel('/api/v1/courses?userId=abc')).toBe('/api/v1/courses'); + }); +}); + +describe('metrics snapshot', () => { + beforeEach(() => { + metricsCollector.reset(); + workerRegistry.reset(); + }); + + it('exposes a versioned schema with cache, http, error and worker sections', () => { + metricsCollector.recordRequest('GET', '/api/v1/certificates/4242', 25, 200); + metricsCollector.recordError('ValidationError', 'student email is invalid', 400); + workerRegistry.register('storage-pin', { concurrency: 10 }); + + const snapshot = buildMetricsSnapshot(); + + expect(snapshot.schemaVersion).toBe(METRICS_SCHEMA_VERSION); + expect(snapshot.http.requestsTotal).toBe(1); + expect(snapshot.http.requestsByRoute).toEqual({ 'GET /api/v1/certificates/:id': 1 }); + expect(snapshot.http.requestsByStatusClass).toEqual({ '2xx': 1 }); + expect(snapshot.errors.errorsByType).toEqual({ ValidationError: 1 }); + expect(snapshot.cache).toEqual( + expect.objectContaining({ hitsTotal: expect.any(Number), hitRatio: expect.any(Number) }) + ); + expect(snapshot.workers).toEqual([ + expect.objectContaining({ name: 'storage-pin', state: 'running', concurrency: 10 }), + ]); + }); + + it('never carries error messages or request payloads', () => { + metricsCollector.recordError('ValidationError', 'student email is invalid', 400); + metricsCollector.recordEvent('user.registered', { email: 'leak@example.com' }); + + const serialized = JSON.stringify(buildMetricsSnapshot()); + + expect(serialized).not.toContain('student email is invalid'); + expect(serialized).not.toContain('leak@example.com'); + }); +}); + +describe('prometheus rendering', () => { + beforeEach(() => { + metricsCollector.reset(); + workerRegistry.reset(); + }); + + it('emits HELP and TYPE lines with units in the metric names', () => { + metricsCollector.recordRequest('GET', '/api/v1/courses', 10, 200); + workerRegistry.register('webhook-delivery'); + workerRegistry.recordFailed('webhook-delivery'); + + const text = renderPrometheus(); + + expect(text).toContain('# HELP w3sl_cache_hit_ratio'); + expect(text).toContain('# TYPE w3sl_cache_hits_total counter'); + expect(text).toContain('w3sl_http_requests_total{method="GET",route="/api/v1/courses"} 1'); + expect(text).toContain('w3sl_http_responses_total{status_class="2xx"} 1'); + expect(text).toContain('w3sl_worker_up{worker="webhook-delivery",state="running"} 1'); + expect(text).toContain('w3sl_worker_jobs_failed_total{worker="webhook-delivery"} 1'); + expect(text).toContain('w3sl_process_uptime_seconds'); + expect(text.endsWith('\n')).toBe(true); + }); + + it('escapes label values so exposition output cannot be broken', () => { + metricsCollector.recordError('Weird"Error\nType', 'ignored'); + + const text = renderPrometheus(); + + expect(text).toContain('w3sl_errors_total{type="Weird\\"Error Type"} 1'); + }); +}); + +describe('metrics endpoints', () => { + const app = express(); + app.use(express.json()); + app.use('/metrics', metricsRouter); + + beforeEach(() => { + metricsCollector.reset(); + workerRegistry.reset(); + }); + + it('serves the prometheus exposition as text/plain', async () => { + const res = await request(app).get('/metrics/prometheus'); + + expect(res.status).toBe(200); + expect(res.headers['content-type']).toContain('text/plain'); + expect(res.text).toContain('# TYPE w3sl_process_uptime_seconds gauge'); + }); + + it('serves the JSON snapshot with its schema version', async () => { + const res = await request(app).get('/metrics/snapshot'); + + expect(res.status).toBe(200); + expect(res.body.schemaVersion).toBe(METRICS_SCHEMA_VERSION); + expect(res.body.data).toHaveProperty('cache'); + expect(res.body.data).toHaveProperty('workers'); + }); + + it('redacts error messages from the raw error listing', async () => { + metricsCollector.recordError('ValidationError', 'student email is invalid', 400); + + const res = await request(app).get('/metrics/errors'); + + expect(res.status).toBe(200); + expect(res.body.data[0]).toEqual({ + type: 'ValidationError', + statusCode: 400, + timestamp: expect.any(String), + }); + expect(JSON.stringify(res.body)).not.toContain('student email is invalid'); + }); +}); + +describe('requireMetricsAuth', () => { + const buildGuardedApp = () => { + const app = express(); + app.get('/guarded', requireMetricsAuth, (_req, res) => res.json({ ok: true })); + return app; + }; + + const originalToken = process.env.METRICS_AUTH_TOKEN; + + afterEach(() => { + if (originalToken === undefined) { + delete process.env.METRICS_AUTH_TOKEN; + } else { + process.env.METRICS_AUTH_TOKEN = originalToken; + } + }); + + it('rejects requests without the configured token', async () => { + process.env.METRICS_AUTH_TOKEN = 'super-secret'; + + const res = await request(buildGuardedApp()).get('/guarded'); + + expect(res.status).toBe(401); + expect(res.body.error.code).toBe('UNAUTHORIZED'); + expect(res.body.error.requestId).toBeTruthy(); + }); + + it('accepts the token via X-Metrics-Token', async () => { + process.env.METRICS_AUTH_TOKEN = 'super-secret'; + + const res = await request(buildGuardedApp()) + .get('/guarded') + .set('X-Metrics-Token', 'super-secret'); + + expect(res.status).toBe(200); + }); + + it('accepts the token via Authorization: Bearer', async () => { + process.env.METRICS_AUTH_TOKEN = 'super-secret'; + + const res = await request(buildGuardedApp()) + .get('/guarded') + .set('Authorization', 'Bearer super-secret'); + + expect(res.status).toBe(200); + }); + + it('rejects a wrong token', async () => { + process.env.METRICS_AUTH_TOKEN = 'super-secret'; + + const res = await request(buildGuardedApp()) + .get('/guarded') + .set('X-Metrics-Token', 'nope'); + + expect(res.status).toBe(401); + }); +});