diff --git a/src/__tests__/kafkaMonitoring.test.ts b/src/__tests__/kafkaMonitoring.test.ts new file mode 100644 index 0000000..3bbabdd --- /dev/null +++ b/src/__tests__/kafkaMonitoring.test.ts @@ -0,0 +1,335 @@ +// Unit tests for Kafka consumer lag monitoring and auto-scaling. +// Issue #109 — comprehensive test suite. + +import { describe, it, expect } from 'vitest'; +import { + derivePartitionStatus, + deriveGroupStatus, + computeTargetInstances, + buildScalingEvent, + generateDemoSnapshot, + generateDemoScalingStatuses, + createDemoKafkaMonitoringService, +} from '../services/kafkaMonitoringService'; +import type { ConsumerGroupScalingConfig, PartitionLag } from '../types/kafka'; + +// ── derivePartitionStatus ──────────────────────────────────────────────────── + +describe('derivePartitionStatus', () => { + it('returns healthy for lag below warning threshold', () => { + expect(derivePartitionStatus(0)).toBe('healthy'); + expect(derivePartitionStatus(500)).toBe('healthy'); + expect(derivePartitionStatus(999)).toBe('healthy'); + }); + + it('returns warning for lag between thresholds', () => { + expect(derivePartitionStatus(1_000)).toBe('warning'); + expect(derivePartitionStatus(5_000)).toBe('warning'); + expect(derivePartitionStatus(9_999)).toBe('warning'); + }); + + it('returns critical for lag at or above critical threshold', () => { + expect(derivePartitionStatus(10_000)).toBe('critical'); + expect(derivePartitionStatus(100_000)).toBe('critical'); + }); +}); + +// ── deriveGroupStatus ──────────────────────────────────────────────────────── + +describe('deriveGroupStatus', () => { + function makePartition(lag: number): PartitionLag { + return { + partition: 0, + logEndOffset: lag + 1000, + consumerOffset: 1000, + lag, + status: derivePartitionStatus(lag), + }; + } + + it('returns healthy when all partitions are healthy', () => { + const partitions = [makePartition(0), makePartition(100), makePartition(500)]; + expect(deriveGroupStatus(partitions)).toBe('healthy'); + }); + + it('returns warning when any partition is warning', () => { + const partitions = [makePartition(0), makePartition(2_000), makePartition(100)]; + expect(deriveGroupStatus(partitions)).toBe('warning'); + }); + + it('returns critical when any partition is critical', () => { + const partitions = [makePartition(0), makePartition(2_000), makePartition(15_000)]; + expect(deriveGroupStatus(partitions)).toBe('critical'); + }); + + it('critical takes precedence over warning', () => { + const partitions = [makePartition(5_000), makePartition(20_000)]; + expect(deriveGroupStatus(partitions)).toBe('critical'); + }); +}); + +// ── computeTargetInstances ─────────────────────────────────────────────────── + +describe('computeTargetInstances', () => { + const base: ConsumerGroupScalingConfig = { + groupId: 'test-group', + scaleOutLagThreshold: 5_000, + scaleInLagThreshold: 200, + minInstances: 1, + maxInstances: 8, + currentInstances: 3, + }; + + it('returns null when lag is within stable range', () => { + expect(computeTargetInstances(1_000, base)).toBeNull(); + expect(computeTargetInstances(200, base)).toBeNull(); + }); + + it('scales out when lag exceeds scaleOutLagThreshold', () => { + const result = computeTargetInstances(10_000, base); + expect(result).not.toBeNull(); + expect(result!).toBeGreaterThan(base.currentInstances); + expect(result!).toBeLessThanOrEqual(base.maxInstances); + }); + + it('scales in when lag drops below scaleInLagThreshold', () => { + const result = computeTargetInstances(50, base); + expect(result).not.toBeNull(); + expect(result!).toBeLessThan(base.currentInstances); + expect(result!).toBeGreaterThanOrEqual(base.minInstances); + }); + + it('does not scale out beyond maxInstances', () => { + const atMax = { ...base, currentInstances: 8 }; + expect(computeTargetInstances(100_000, atMax)).toBeNull(); + }); + + it('does not scale in below minInstances', () => { + const atMin = { ...base, currentInstances: 1 }; + expect(computeTargetInstances(0, atMin)).toBeNull(); + }); + + it('scale-out factor grows with lag magnitude', () => { + const result1 = computeTargetInstances(5_000, base); + const result2 = computeTargetInstances(15_000, base); + // higher lag should produce a higher or equal target + expect((result2 ?? 0)).toBeGreaterThanOrEqual(result1 ?? 0); + }); +}); + +// ── buildScalingEvent ──────────────────────────────────────────────────────── + +describe('buildScalingEvent', () => { + it('produces a valid ScalingEvent', () => { + const event = buildScalingEvent('my-group', 2, 4, 8_000); + expect(event.groupId).toBe('my-group'); + expect(event.previousInstances).toBe(2); + expect(event.targetInstances).toBe(4); + expect(event.lagAtTrigger).toBe(8_000); + expect(event.reason).toBe('lag-high'); + expect(typeof event.id).toBe('string'); + expect(event.id.length).toBeGreaterThan(0); + expect(typeof event.triggeredAt).toBe('number'); + }); + + it('sets reason to lag-low when scaling in', () => { + const event = buildScalingEvent('my-group', 4, 2, 100); + expect(event.reason).toBe('lag-low'); + }); + + it('sets reason to manual when specified', () => { + const event = buildScalingEvent('my-group', 3, 3, 0, 'manual'); + expect(event.reason).toBe('manual'); + }); + + it('generates unique IDs for distinct calls', () => { + const e1 = buildScalingEvent('g', 1, 2, 0); + const e2 = buildScalingEvent('g', 1, 2, 0); + expect(e1.id).not.toBe(e2.id); + }); +}); + +// ── generateDemoSnapshot ───────────────────────────────────────────────────── + +describe('generateDemoSnapshot', () => { + it('returns an array of ConsumerGroupLag objects', () => { + const snapshot = generateDemoSnapshot(); + expect(Array.isArray(snapshot)).toBe(true); + expect(snapshot.length).toBeGreaterThan(0); + }); + + it('each group has required fields', () => { + const snapshot = generateDemoSnapshot(); + for (const group of snapshot) { + expect(typeof group.groupId).toBe('string'); + expect(typeof group.topic).toBe('string'); + expect(Array.isArray(group.partitions)).toBe(true); + expect(group.partitions.length).toBeGreaterThan(0); + expect(typeof group.totalLag).toBe('number'); + expect(typeof group.maxPartitionLag).toBe('number'); + expect(typeof group.capturedAt).toBe('number'); + expect(['healthy', 'warning', 'critical']).toContain(group.status); + } + }); + + it('totalLag equals the sum of partition lags', () => { + const snapshot = generateDemoSnapshot(); + for (const group of snapshot) { + const sum = group.partitions.reduce((s, p) => s + p.lag, 0); + expect(group.totalLag).toBe(sum); + } + }); + + it('maxPartitionLag equals the max of individual partition lags', () => { + const snapshot = generateDemoSnapshot(); + for (const group of snapshot) { + const max = Math.max(...group.partitions.map((p) => p.lag)); + expect(group.maxPartitionLag).toBe(max); + } + }); + + it('partition lag >= 0 for all partitions', () => { + const snapshot = generateDemoSnapshot(); + for (const group of snapshot) { + for (const p of group.partitions) { + expect(p.lag).toBeGreaterThanOrEqual(0); + } + } + }); +}); + +// ── generateDemoScalingStatuses ────────────────────────────────────────────── + +describe('generateDemoScalingStatuses', () => { + it('returns statuses for each demo group', () => { + const statuses = generateDemoScalingStatuses(); + expect(statuses.length).toBeGreaterThan(0); + for (const s of statuses) { + expect(typeof s.groupId).toBe('string'); + expect(typeof s.enabled).toBe('boolean'); + expect(typeof s.config.minInstances).toBe('number'); + expect(typeof s.config.maxInstances).toBe('number'); + expect(s.config.minInstances).toBeGreaterThanOrEqual(1); + expect(s.config.maxInstances).toBeGreaterThanOrEqual(s.config.minInstances); + } + }); +}); + +// ── createDemoKafkaMonitoringService ───────────────────────────────────────── + +describe('createDemoKafkaMonitoringService', () => { + it('fetchConsumerLag resolves to a non-empty array', async () => { + const svc = createDemoKafkaMonitoringService(); + const result = await svc.fetchConsumerLag(); + expect(Array.isArray(result)).toBe(true); + expect(result.length).toBeGreaterThan(0); + }); + + it('fetchScalingStatuses resolves to a non-empty array', async () => { + const svc = createDemoKafkaMonitoringService(); + const result = await svc.fetchScalingStatuses(); + expect(Array.isArray(result)).toBe(true); + expect(result.length).toBeGreaterThan(0); + }); + + it('triggerManualScale updates the instance count', async () => { + const svc = createDemoKafkaMonitoringService(); + const statuses = await svc.fetchScalingStatuses(); + const groupId = statuses[0].groupId; + + const event = await svc.triggerManualScale(groupId, 5); + expect(event.groupId).toBe(groupId); + expect(event.targetInstances).toBe(5); + expect(event.reason).toBe('manual'); + + // Config should reflect the new instance count + const updated = await svc.fetchScalingStatuses(); + const updatedStatus = updated.find((s) => s.groupId === groupId); + expect(updatedStatus?.config.currentInstances).toBe(5); + }); +}); + +// ── Kafka store ────────────────────────────────────────────────────────────── + +describe('kafkaSlice store', () => { + // Dynamic import to get a fresh module-level store reference + it('upsertGroupLag and setGroups update the store correctly', async () => { + const { useKafkaStore } = await import('../store/kafkaSlice'); + useKafkaStore.getState().reset(); + + const group = generateDemoSnapshot()[0]; + useKafkaStore.getState().upsertGroupLag(group); + + const state = useKafkaStore.getState(); + expect(state.groups[group.groupId]).toBeDefined(); + expect(state.groups[group.groupId].groupId).toBe(group.groupId); + }); + + it('addScalingEvent prepends to history and respects max 200', async () => { + const { useKafkaStore } = await import('../store/kafkaSlice'); + useKafkaStore.getState().reset(); + + for (let i = 0; i < 205; i++) { + useKafkaStore.getState().addScalingEvent( + buildScalingEvent('g', 1, 2, 0), + ); + } + + expect(useKafkaStore.getState().scalingHistory.length).toBe(200); + }); + + it('markRefreshed sets isLoaded and lastRefreshedAt', async () => { + const { useKafkaStore } = await import('../store/kafkaSlice'); + useKafkaStore.getState().reset(); + + const before = Date.now(); + useKafkaStore.getState().markRefreshed(); + const state = useKafkaStore.getState(); + + expect(state.isLoaded).toBe(true); + expect(state.lastRefreshedAt).toBeGreaterThanOrEqual(before); + expect(state.error).toBeNull(); + }); + + it('setError records error message', async () => { + const { useKafkaStore } = await import('../store/kafkaSlice'); + useKafkaStore.getState().reset(); + + useKafkaStore.getState().setError('something broke'); + expect(useKafkaStore.getState().error).toBe('something broke'); + }); + + it('reset returns store to initial state', async () => { + const { useKafkaStore } = await import('../store/kafkaSlice'); + useKafkaStore.getState().markRefreshed(); + useKafkaStore.getState().reset(); + + const state = useKafkaStore.getState(); + expect(state.isLoaded).toBe(false); + expect(state.groups).toEqual({}); + expect(state.scalingHistory).toEqual([]); + expect(state.error).toBeNull(); + }); +}); + +// ── Performance guard: computeTargetInstances P99 < 100ms ─────────────────── + +describe('performance', () => { + it('computeTargetInstances completes 10 000 evaluations in < 100ms', () => { + const config: ConsumerGroupScalingConfig = { + groupId: 'perf-group', + scaleOutLagThreshold: 5_000, + scaleInLagThreshold: 200, + minInstances: 1, + maxInstances: 16, + currentInstances: 4, + }; + + const start = performance.now(); + for (let i = 0; i < 10_000; i++) { + computeTargetInstances(i * 10, config); + } + const elapsed = performance.now() - start; + expect(elapsed).toBeLessThan(100); + }); +}); diff --git a/src/components/kafka/ConsumerGroupLagCard.tsx b/src/components/kafka/ConsumerGroupLagCard.tsx new file mode 100644 index 0000000..99bc188 --- /dev/null +++ b/src/components/kafka/ConsumerGroupLagCard.tsx @@ -0,0 +1,196 @@ +'use client'; + +// ConsumerGroupLagCard — per-group lag breakdown card. +// Issue #109 — Kafka Consumer Lag Monitoring. + +import { useState } from 'react'; +import { LagStatusBadge } from './LagStatusBadge'; +import type { ConsumerGroupLag, ScalingStatus } from '../../types/kafka'; + +interface ConsumerGroupLagCardProps { + group: ConsumerGroupLag; + scalingStatus?: ScalingStatus; + onManualScale?: (groupId: string, targetInstances: number) => void; +} + +function formatLag(lag: number): string { + if (lag >= 1_000_000) return `${(lag / 1_000_000).toFixed(2)}M`; + if (lag >= 1_000) return `${(lag / 1_000).toFixed(1)}K`; + return String(lag); +} + +function formatTime(ts: number): string { + return new Date(ts).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +export function ConsumerGroupLagCard({ + group, + scalingStatus, + onManualScale, +}: ConsumerGroupLagCardProps) { + const [expanded, setExpanded] = useState(false); + const [scaleInput, setScaleInput] = useState(''); + + const { groupId, topic, partitions, totalLag, maxPartitionLag, capturedAt, status } = group; + const config = scalingStatus?.config; + + function handleManualScale() { + const target = parseInt(scaleInput, 10); + if (!isNaN(target) && target > 0 && onManualScale) { + onManualScale(groupId, target); + setScaleInput(''); + } + } + + return ( +
+ {/* Header */} +
+
+

+ Consumer Group +

+

{groupId}

+

Topic: {topic}

+
+ +
+ + {/* Summary stats */} +
+ + + + {config && ( + + )} +
+ + {/* Partition table toggle */} + + + {/* Partition breakdown */} + {expanded && ( +
+ + + + + + + + + + + + {partitions.map((p) => ( + + + + + + + + ))} + +
PartitionLog-End OffsetConsumer OffsetLagStatus
{p.partition} + {p.logEndOffset.toLocaleString()} + + {p.consumerOffset.toLocaleString()} + + {formatLag(p.lag)} + + +
+
+ )} + + {/* Manual scaling */} + {scalingStatus?.enabled && onManualScale && config && ( +
+ + setScaleInput(e.target.value)} + placeholder={String(config.currentInstances)} + className="w-20 rounded-lg border border-white/10 bg-slate-800 px-2 py-1 text-xs text-white placeholder-slate-600 focus:outline-none focus:ring-2 focus:ring-amber-400" + aria-label={`Target instance count for ${groupId}`} + /> + + + range {config.minInstances}–{config.maxInstances} + +
+ )} + + {/* Last event */} + {scalingStatus?.lastEvent && ( +

+ Last scaling:{' '} + + {scalingStatus.lastEvent.previousInstances} → {scalingStatus.lastEvent.targetInstances} instances + {' '} + at {formatTime(scalingStatus.lastEvent.triggeredAt)} +

+ )} + +

Snapshot at {formatTime(capturedAt)}

+
+ ); +} + +function StatCell({ + label, + value, + highlight = false, +}: { + label: string; + value: string; + highlight?: boolean; +}) { + return ( +
+

{label}

+

+ {value} +

+
+ ); +} diff --git a/src/components/kafka/KafkaMonitoringDashboard.tsx b/src/components/kafka/KafkaMonitoringDashboard.tsx new file mode 100644 index 0000000..102ac2e --- /dev/null +++ b/src/components/kafka/KafkaMonitoringDashboard.tsx @@ -0,0 +1,129 @@ +'use client'; + +// KafkaMonitoringDashboard — top-level Kafka consumer lag & auto-scaling dashboard. +// Issue #109 — system-wide implementation. +// +// Renders: +// • A summary row (total groups, critical count, healthy count) +// • A card per consumer group (ConsumerGroupLagCard) +// • A scaling-event history table (ScalingHistoryTable) +// • Live refresh indicator and manual refresh button + +import { useKafkaMonitoring } from '../../hooks/useKafkaMonitoring'; +import { ConsumerGroupLagCard } from './ConsumerGroupLagCard'; +import { ScalingHistoryTable } from './ScalingHistoryTable'; +import { LagStatusBadge } from './LagStatusBadge'; + +function formatLastRefreshed(ts: number | null): string { + if (!ts) return 'Never'; + return new Date(ts).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +export function KafkaMonitoringDashboard() { + const { + groups, + scalingStatus, + scalingHistory, + isLoaded, + error, + lastRefreshedAt, + triggerManualScale, + refresh, + } = useKafkaMonitoring(); + + const groupList = Object.values(groups); + const criticalCount = groupList.filter((g) => g.status === 'critical').length; + const warningCount = groupList.filter((g) => g.status === 'warning').length; + const healthyCount = groupList.filter((g) => g.status === 'healthy').length; + + return ( +
+ {/* Dashboard header */} +
+
+

+ Kafka Consumer Lag Monitoring +

+

+ Auto-scaling consumer groups — issue #109 +

+
+ +
+ {/* Summary pills */} + {groupList.length > 0 && ( + <> + {criticalCount > 0 && ( + + )} + {warningCount > 0 && ( + + )} + + + )} + + {/* Refresh controls */} +
+ Last refresh: {formatLastRefreshed(lastRefreshedAt)} + +
+
+
+ + {/* Error banner */} + {error && ( +
+ {error} +
+ )} + + {/* Loading state */} + {!isLoaded && !error && ( +
+
+ )} + + {/* Group cards */} + {isLoaded && groupList.length === 0 && !error && ( +

+ No consumer groups found. +

+ )} + + {isLoaded && groupList.length > 0 && ( +
+ {groupList.map((group) => ( + + ))} +
+ )} + + {/* Scaling history */} + {isLoaded && ( + + )} +
+ ); +} diff --git a/src/components/kafka/LagStatusBadge.tsx b/src/components/kafka/LagStatusBadge.tsx new file mode 100644 index 0000000..f8a38a8 --- /dev/null +++ b/src/components/kafka/LagStatusBadge.tsx @@ -0,0 +1,38 @@ +'use client'; + +// LagStatusBadge — colour-coded pill indicating consumer group / partition health. +// Issue #109 — Kafka Consumer Lag Monitoring. + +import type { PartitionLagStatus } from '../../types/kafka'; + +interface LagStatusBadgeProps { + status: PartitionLagStatus; + /** Accessible label override. Defaults to the status string. */ + label?: string; +} + +const STATUS_STYLES: Record = { + healthy: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30', + warning: 'bg-amber-400/15 text-amber-400 border-amber-400/30', + critical: 'bg-rose-500/15 text-rose-400 border-rose-500/30', +}; + +const STATUS_DOT: Record = { + healthy: 'bg-emerald-400', + warning: 'bg-amber-400 animate-pulse', + critical: 'bg-rose-500 animate-pulse', +}; + +export function LagStatusBadge({ status, label }: LagStatusBadgeProps) { + const displayLabel = label ?? status.charAt(0).toUpperCase() + status.slice(1); + return ( + + + ); +} diff --git a/src/components/kafka/ScalingHistoryTable.tsx b/src/components/kafka/ScalingHistoryTable.tsx new file mode 100644 index 0000000..fda6150 --- /dev/null +++ b/src/components/kafka/ScalingHistoryTable.tsx @@ -0,0 +1,94 @@ +'use client'; + +// ScalingHistoryTable — compact table of recent auto-scaling events. +// Issue #109 — Kafka Consumer Lag Monitoring. + +import type { ScalingEvent } from '../../types/kafka'; + +interface ScalingHistoryTableProps { + events: ScalingEvent[]; + /** Maximum rows to display. Defaults to 20. */ + maxRows?: number; +} + +const REASON_LABEL: Record = { + 'lag-high': '↑ Lag high', + 'lag-low': '↓ Lag low', + manual: '⚙ Manual', +}; + +const REASON_COLOR: Record = { + 'lag-high': 'text-rose-400', + 'lag-low': 'text-emerald-400', + manual: 'text-amber-400', +}; + +function formatTs(ts: number): string { + return new Date(ts).toLocaleString([], { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +function formatLag(lag: number): string { + if (lag >= 1_000_000) return `${(lag / 1_000_000).toFixed(2)}M`; + if (lag >= 1_000) return `${(lag / 1_000).toFixed(1)}K`; + return String(lag); +} + +export function ScalingHistoryTable({ + events, + maxRows = 20, +}: ScalingHistoryTableProps) { + const visible = events.slice(0, maxRows); + + return ( +
+

Scaling History

+ + {visible.length === 0 ? ( +

+ No scaling events recorded yet. +

+ ) : ( +
+ + + + + + + + + + + + {visible.map((ev) => ( + + + + + + + + ))} + +
TimeGroupInstancesLag at TriggerReason
{formatTs(ev.triggeredAt)}{ev.groupId} + {ev.previousInstances} → {ev.targetInstances} + {formatLag(ev.lagAtTrigger)} + {REASON_LABEL[ev.reason]} +
+
+ )} +
+ ); +} diff --git a/src/hooks/useKafkaMonitoring.ts b/src/hooks/useKafkaMonitoring.ts new file mode 100644 index 0000000..0fafe51 --- /dev/null +++ b/src/hooks/useKafkaMonitoring.ts @@ -0,0 +1,155 @@ +'use client'; + +// useKafkaMonitoring — Kafka consumer lag polling and auto-scaling hook. +// Issue #109 — system-wide implementation. +// +// Polls the Kafka monitoring service on a configurable interval, writes +// snapshots into the Zustand kafkaSlice, and evaluates auto-scaling decisions +// locally so the UI reflects the latest state without a round-trip. +// +// Performance target: < 100ms P99 for the synchronous evaluation path. + +import { useEffect, useRef, useCallback } from 'react'; +import { useKafkaStore } from '../store/kafkaSlice'; +import { + createDemoKafkaMonitoringService, + computeTargetInstances, + buildScalingEvent, +} from '../services/kafkaMonitoringService'; +import type { KafkaMonitoringProvider } from '../services/kafkaMonitoringService'; + +/** Default poll interval in milliseconds. */ +const DEFAULT_POLL_INTERVAL_MS = 15_000; + +export interface UseKafkaMonitoringOptions { + /** Set to false to pause polling. Defaults to true. */ + enabled?: boolean; + /** Poll interval in ms. Defaults to 15 000. */ + pollIntervalMs?: number; + /** + * Optional custom provider. Falls back to the demo service when omitted, + * matching the project-wide pattern of demo-first development. + */ + provider?: KafkaMonitoringProvider; +} + +export function useKafkaMonitoring(options: UseKafkaMonitoringOptions = {}) { + const { + enabled = true, + pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, + provider, + } = options; + + const store = useKafkaStore(); + const serviceRef = useRef( + provider ?? createDemoKafkaMonitoringService(), + ); + const timerRef = useRef | undefined>(undefined); + const mountedRef = useRef(true); + + const poll = useCallback(async () => { + const svc = serviceRef.current; + + try { + // Fetch lag snapshots and scaling statuses in parallel. + const [groups, statuses] = await Promise.all([ + svc.fetchConsumerLag(), + svc.fetchScalingStatuses(), + ]); + + if (!mountedRef.current) return; + + store.setGroups(groups); + statuses.forEach((s) => store.upsertScalingStatus(s)); + store.markRefreshed(); + + // Evaluate auto-scaling decisions for each group. + for (const status of statuses) { + if (!status.enabled) continue; + + const groupLag = groups.find((g) => g.groupId === status.groupId); + if (!groupLag) continue; + + const target = computeTargetInstances(groupLag.totalLag, status.config); + if (target === null) continue; + + const event = buildScalingEvent( + status.groupId, + status.config.currentInstances, + target, + groupLag.totalLag, + ); + + // Persist to store; production would also call svc.triggerManualScale(). + store.addScalingEvent(event); + store.upsertScalingStatus({ + ...status, + config: { ...status.config, currentInstances: target }, + lastEvent: event, + }); + } + } catch (err) { + if (!mountedRef.current) return; + store.setError(err instanceof Error ? err.message : 'Kafka poll failed'); + } + }, [store]); + + // Trigger an immediate manual scale action. + const triggerManualScale = useCallback( + async (groupId: string, targetInstances: number) => { + try { + const event = await serviceRef.current.triggerManualScale(groupId, targetInstances); + store.addScalingEvent(event); + + const existing = useKafkaStore.getState().scalingStatus[groupId]; + if (existing) { + store.upsertScalingStatus({ + ...existing, + config: { ...existing.config, currentInstances: targetInstances }, + lastEvent: event, + }); + } + } catch (err) { + store.setError( + err instanceof Error ? err.message : 'Manual scale request failed', + ); + } + }, + [store], + ); + + useEffect(() => { + if (!enabled) return; + + mountedRef.current = true; + + // Immediate first poll, then schedule repeating interval. + void poll(); + + function scheduleNext() { + timerRef.current = setTimeout(async () => { + await poll(); + if (mountedRef.current) scheduleNext(); + }, pollIntervalMs); + } + scheduleNext(); + + return () => { + mountedRef.current = false; + if (timerRef.current !== undefined) { + clearTimeout(timerRef.current); + } + }; + }, [enabled, pollIntervalMs, poll]); + + return { + groups: store.groups, + scalingStatus: store.scalingStatus, + scalingHistory: store.scalingHistory, + isLoaded: store.isLoaded, + error: store.error, + lastRefreshedAt: store.lastRefreshedAt, + triggerManualScale, + refresh: poll, + }; +} diff --git a/src/services/kafkaMonitoringService.ts b/src/services/kafkaMonitoringService.ts new file mode 100644 index 0000000..da52982 --- /dev/null +++ b/src/services/kafkaMonitoringService.ts @@ -0,0 +1,250 @@ +// Kafka consumer lag monitoring service. +// Issue #109 — system-wide implementation. +// +// In production this module would call a real Kafka metrics endpoint (e.g. +// Burrow, Confluent REST Proxy, or a bespoke scraper). In demo mode it returns +// deterministic synthetic data so the UI can be developed and tested without +// a live Kafka cluster. + +import type { + ConsumerGroupLag, + ConsumerGroupScalingConfig, + PartitionLag, + PartitionLagStatus, + ScalingEvent, + ScalingStatus, +} from '../types/kafka'; + +// ── Lag status thresholds ──────────────────────────────────────────────────── + +/** Per-partition lag that triggers a "warning" status. */ +const WARNING_LAG_THRESHOLD = 1_000; +/** Per-partition lag that triggers a "critical" status. */ +const CRITICAL_LAG_THRESHOLD = 10_000; + +export function derivePartitionStatus(lag: number): PartitionLagStatus { + if (lag >= CRITICAL_LAG_THRESHOLD) return 'critical'; + if (lag >= WARNING_LAG_THRESHOLD) return 'warning'; + return 'healthy'; +} + +export function deriveGroupStatus(partitions: PartitionLag[]): PartitionLagStatus { + if (partitions.some((p) => p.status === 'critical')) return 'critical'; + if (partitions.some((p) => p.status === 'warning')) return 'warning'; + return 'healthy'; +} + +// ── Auto-scaling logic ──────────────────────────────────────────────────────── + +/** + * Given current lag and a scaling config, compute the target instance count. + * Returns `null` when no change is required. + * + * Performance target: pure sync function, always < 100 µs. + */ +export function computeTargetInstances( + totalLag: number, + config: ConsumerGroupScalingConfig, +): number | null { + const { currentInstances, minInstances, maxInstances } = config; + + if (totalLag >= config.scaleOutLagThreshold && currentInstances < maxInstances) { + // Scale out: add one instance per 10× lag above threshold, capped at max. + const factor = Math.ceil(totalLag / config.scaleOutLagThreshold); + return Math.min(currentInstances + factor, maxInstances); + } + + if (totalLag < config.scaleInLagThreshold && currentInstances > minInstances) { + // Scale in: remove one instance. + return Math.max(currentInstances - 1, minInstances); + } + + return null; // no change +} + +/** + * Build a ScalingEvent from a scaling decision. + */ +export function buildScalingEvent( + groupId: string, + previousInstances: number, + targetInstances: number, + lagAtTrigger: number, + reason: ScalingEvent['reason'] = 'lag-high', +): ScalingEvent { + let derivedReason: ScalingEvent['reason']; + if (reason === 'manual') { + derivedReason = 'manual'; + } else { + derivedReason = targetInstances > previousInstances ? 'lag-high' : 'lag-low'; + } + + return { + id: `${groupId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + groupId, + triggeredAt: Date.now(), + previousInstances, + targetInstances, + reason: derivedReason, + lagAtTrigger, + }; +} + +// ── Demo data generation ────────────────────────────────────────────────────── + +/** Lightweight LCG for reproducible-ish demo data that evolves each call. */ +let _seed = 0xdeadbeef; +function lcg(): number { + _seed = Math.imul(_seed ^ (_seed >>> 13), 0x9e3779b9); + _seed = _seed ^ (_seed >>> 11); + return (_seed >>> 0) / 0x1_0000_0000; +} + +const DEMO_TOPICS = [ + 'validator-attestations', + 'block-proposals', + 'sync-committee-duties', + 'reward-calculations', + 'node-health-events', +]; + +const DEMO_GROUPS = [ + 'attestation-processor', + 'block-builder', + 'sync-committee-monitor', + 'reward-indexer', + 'health-aggregator', +]; + +const DEMO_SCALING_CONFIGS: ConsumerGroupScalingConfig[] = DEMO_GROUPS.map((groupId, i) => ({ + groupId, + scaleOutLagThreshold: 5_000, + scaleInLagThreshold: 200, + minInstances: 1, + maxInstances: 8, + currentInstances: 2 + (i % 3), +})); + +/** Generate a deterministic-ish snapshot for all demo consumer groups. */ +export function generateDemoSnapshot(): ConsumerGroupLag[] { + return DEMO_GROUPS.map((groupId, gi) => { + const topic = DEMO_TOPICS[gi % DEMO_TOPICS.length]; + const partitionCount = 4 + (gi % 4); // 4–7 partitions per group + const partitions: PartitionLag[] = Array.from({ length: partitionCount }, (_, pi) => { + const logEndOffset = 100_000 + Math.floor(lcg() * 50_000); + // Occasionally spike one partition for demo variety + const lagBase = gi === 2 && pi === 1 ? 12_000 : gi === 0 ? 800 : 200; + const lag = Math.max(0, Math.floor(lagBase + lcg() * lagBase)); + const consumerOffset = logEndOffset - lag; + const status = derivePartitionStatus(lag); + return { partition: pi, logEndOffset, consumerOffset, lag, status }; + }); + + const totalLag = partitions.reduce((s, p) => s + p.lag, 0); + const maxPartitionLag = Math.max(...partitions.map((p) => p.lag)); + + return { + groupId, + topic, + partitions, + totalLag, + maxPartitionLag, + capturedAt: Date.now(), + status: deriveGroupStatus(partitions), + }; + }); +} + +/** Generate initial scaling status for all demo groups. */ +export function generateDemoScalingStatuses(): ScalingStatus[] { + return DEMO_SCALING_CONFIGS.map((config) => ({ + groupId: config.groupId, + enabled: true, + config, + lastEvent: null, + })); +} + +// ── HTTP service ────────────────────────────────────────────────────────────── + +export interface KafkaMonitoringProvider { + /** + * Fetch the current consumer lag snapshot for all tracked groups. + * Returns an empty array on any error. + */ + fetchConsumerLag(): Promise; + + /** + * Fetch current scaling status for all groups. + */ + fetchScalingStatuses(): Promise; + + /** + * Trigger a manual scaling action for a group. + * Returns the resulting ScalingEvent. + */ + triggerManualScale(groupId: string, targetInstances: number): Promise; +} + +/** Demo provider — deterministic, zero network calls. */ +export function createDemoKafkaMonitoringService(): KafkaMonitoringProvider { + const scalingStatuses = generateDemoScalingStatuses(); + + return { + async fetchConsumerLag() { + return generateDemoSnapshot(); + }, + + async fetchScalingStatuses() { + return scalingStatuses; + }, + + async triggerManualScale(groupId, targetInstances) { + const status = scalingStatuses.find((s) => s.groupId === groupId); + const previous = status?.config.currentInstances ?? 1; + const event = buildScalingEvent(groupId, previous, targetInstances, 0, 'manual'); + if (status) { + status.config.currentInstances = targetInstances; + status.lastEvent = event; + } + return event; + }, + }; +} + +/** Production provider — calls real Kafka metrics endpoints. */ +export function createKafkaMonitoringService(baseUrl: string): KafkaMonitoringProvider { + return { + async fetchConsumerLag() { + try { + const res = await fetch(`${baseUrl}/api/kafka/consumer-lag`); + if (!res.ok) return []; + return (await res.json()) as ConsumerGroupLag[]; + } catch { + return []; + } + }, + + async fetchScalingStatuses() { + try { + const res = await fetch(`${baseUrl}/api/kafka/scaling-status`); + if (!res.ok) return []; + return (await res.json()) as ScalingStatus[]; + } catch { + return []; + } + }, + + async triggerManualScale(groupId, targetInstances) { + const res = await fetch(`${baseUrl}/api/kafka/scale`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ groupId, targetInstances }), + }); + if (!res.ok) { + throw new Error(`Scale request failed: ${res.status}`); + } + return (await res.json()) as ScalingEvent; + }, + }; +} diff --git a/src/store/kafkaSlice.ts b/src/store/kafkaSlice.ts new file mode 100644 index 0000000..22f4336 --- /dev/null +++ b/src/store/kafkaSlice.ts @@ -0,0 +1,71 @@ +// Kafka consumer lag monitoring and auto-scaling store. +// Issue #109 — system-wide implementation. + +import { create } from 'zustand'; +import type { + ConsumerGroupLag, + KafkaMonitoringState, + ScalingEvent, + ScalingStatus, +} from '../types/kafka'; + +interface KafkaMonitoringActions { + /** Replace (or insert) a group lag snapshot. */ + upsertGroupLag: (lag: ConsumerGroupLag) => void; + /** Replace all group lag snapshots at once. */ + setGroups: (groups: ConsumerGroupLag[]) => void; + /** Update scaling status for a group. */ + upsertScalingStatus: (status: ScalingStatus) => void; + /** Prepend a new scaling event to the history ring-buffer (max 200). */ + addScalingEvent: (event: ScalingEvent) => void; + /** Mark store as loaded and record the refresh timestamp. */ + markRefreshed: () => void; + /** Set the current error message. Pass null to clear. */ + setError: (error: string | null) => void; + /** Reset the store to initial state. */ + reset: () => void; +} + +const MAX_SCALING_HISTORY = 200; + +const initialState: KafkaMonitoringState = { + groups: {}, + scalingStatus: {}, + scalingHistory: [], + isLoaded: false, + error: null, + lastRefreshedAt: null, +}; + +export const useKafkaStore = create( + (set) => ({ + ...initialState, + + upsertGroupLag: (lag) => + set((s) => ({ + groups: { ...s.groups, [lag.groupId]: lag }, + })), + + setGroups: (groups) => + set(() => ({ + groups: Object.fromEntries(groups.map((g) => [g.groupId, g])), + })), + + upsertScalingStatus: (status) => + set((s) => ({ + scalingStatus: { ...s.scalingStatus, [status.groupId]: status }, + })), + + addScalingEvent: (event) => + set((s) => ({ + scalingHistory: [event, ...s.scalingHistory].slice(0, MAX_SCALING_HISTORY), + })), + + markRefreshed: () => + set({ isLoaded: true, lastRefreshedAt: Date.now(), error: null }), + + setError: (error) => set({ error }), + + reset: () => set(initialState), + }), +); diff --git a/src/types/kafka.ts b/src/types/kafka.ts new file mode 100644 index 0000000..bf118d2 --- /dev/null +++ b/src/types/kafka.ts @@ -0,0 +1,96 @@ +// Kafka consumer lag monitoring and auto-scaling types. +// Issue #109 — system-wide implementation. + +/** Health status of a single consumer group partition. */ +export type PartitionLagStatus = 'healthy' | 'warning' | 'critical'; + +/** A single partition's lag snapshot. */ +export interface PartitionLag { + /** Kafka partition number (0-indexed). */ + partition: number; + /** Log-end offset — latest offset produced to this partition. */ + logEndOffset: number; + /** Current offset committed by the consumer group. */ + consumerOffset: number; + /** Lag = logEndOffset − consumerOffset. */ + lag: number; + /** Derived status based on lag thresholds. */ + status: PartitionLagStatus; +} + +/** Aggregated lag snapshot for one consumer group across all partitions. */ +export interface ConsumerGroupLag { + /** Unique consumer group ID. */ + groupId: string; + /** Kafka topic being consumed. */ + topic: string; + /** Per-partition breakdowns. */ + partitions: PartitionLag[]; + /** Sum of lag across all partitions. */ + totalLag: number; + /** Max single-partition lag. */ + maxPartitionLag: number; + /** Timestamp (ms) when this snapshot was taken. */ + capturedAt: number; + /** Derived group-level health. */ + status: PartitionLagStatus; +} + +/** Auto-scaling decision record for a consumer group. */ +export interface ScalingEvent { + /** Unique event ID. */ + id: string; + /** Consumer group this event applies to. */ + groupId: string; + /** UTC timestamp (ms) of the scaling action. */ + triggeredAt: number; + /** Consumer instance count before the action. */ + previousInstances: number; + /** Consumer instance count after the action. */ + targetInstances: number; + /** Reason code for the decision. */ + reason: 'lag-high' | 'lag-low' | 'manual'; + /** Total lag that triggered the event. */ + lagAtTrigger: number; +} + +/** Configuration thresholds for auto-scaling a consumer group. */ +export interface ConsumerGroupScalingConfig { + groupId: string; + /** Lag above this value triggers a scale-out. */ + scaleOutLagThreshold: number; + /** Lag below this value triggers a scale-in. */ + scaleInLagThreshold: number; + /** Minimum number of consumer instances. */ + minInstances: number; + /** Maximum number of consumer instances. */ + maxInstances: number; + /** Current running instance count. */ + currentInstances: number; +} + +/** Current auto-scaling status for a consumer group. */ +export interface ScalingStatus { + groupId: string; + /** Whether auto-scaling is enabled for this group. */ + enabled: boolean; + config: ConsumerGroupScalingConfig; + /** Most recent scaling event, if any. */ + lastEvent: ScalingEvent | null; +} + +/** Zustand store state shape for the Kafka monitoring slice. */ +export interface KafkaMonitoringState { + /** All tracked consumer group lag snapshots, keyed by groupId. */ + groups: Record; + /** Scaling status keyed by groupId. */ + scalingStatus: Record; + /** History of scaling events (most-recent first). */ + scalingHistory: ScalingEvent[]; + /** Whether initial data has been loaded. */ + isLoaded: boolean; + /** Last poll error message, if any. */ + error: string | null; + /** Timestamp (ms) of the last successful refresh. */ + lastRefreshedAt: number | null; +}