diff --git a/src/__tests__/kafkaMonitoring.test.ts b/src/__tests__/kafkaMonitoring.test.ts
new file mode 100644
index 0000000..3bbabdd
--- /dev/null
+++ b/src/__tests__/kafkaMonitoring.test.ts
@@ -0,0 +1,335 @@
+// Unit tests for Kafka consumer lag monitoring and auto-scaling.
+// Issue #109 — comprehensive test suite.
+
+import { describe, it, expect } from 'vitest';
+import {
+ derivePartitionStatus,
+ deriveGroupStatus,
+ computeTargetInstances,
+ buildScalingEvent,
+ generateDemoSnapshot,
+ generateDemoScalingStatuses,
+ createDemoKafkaMonitoringService,
+} from '../services/kafkaMonitoringService';
+import type { ConsumerGroupScalingConfig, PartitionLag } from '../types/kafka';
+
+// ── derivePartitionStatus ────────────────────────────────────────────────────
+
+describe('derivePartitionStatus', () => {
+ it('returns healthy for lag below warning threshold', () => {
+ expect(derivePartitionStatus(0)).toBe('healthy');
+ expect(derivePartitionStatus(500)).toBe('healthy');
+ expect(derivePartitionStatus(999)).toBe('healthy');
+ });
+
+ it('returns warning for lag between thresholds', () => {
+ expect(derivePartitionStatus(1_000)).toBe('warning');
+ expect(derivePartitionStatus(5_000)).toBe('warning');
+ expect(derivePartitionStatus(9_999)).toBe('warning');
+ });
+
+ it('returns critical for lag at or above critical threshold', () => {
+ expect(derivePartitionStatus(10_000)).toBe('critical');
+ expect(derivePartitionStatus(100_000)).toBe('critical');
+ });
+});
+
+// ── deriveGroupStatus ────────────────────────────────────────────────────────
+
+describe('deriveGroupStatus', () => {
+ function makePartition(lag: number): PartitionLag {
+ return {
+ partition: 0,
+ logEndOffset: lag + 1000,
+ consumerOffset: 1000,
+ lag,
+ status: derivePartitionStatus(lag),
+ };
+ }
+
+ it('returns healthy when all partitions are healthy', () => {
+ const partitions = [makePartition(0), makePartition(100), makePartition(500)];
+ expect(deriveGroupStatus(partitions)).toBe('healthy');
+ });
+
+ it('returns warning when any partition is warning', () => {
+ const partitions = [makePartition(0), makePartition(2_000), makePartition(100)];
+ expect(deriveGroupStatus(partitions)).toBe('warning');
+ });
+
+ it('returns critical when any partition is critical', () => {
+ const partitions = [makePartition(0), makePartition(2_000), makePartition(15_000)];
+ expect(deriveGroupStatus(partitions)).toBe('critical');
+ });
+
+ it('critical takes precedence over warning', () => {
+ const partitions = [makePartition(5_000), makePartition(20_000)];
+ expect(deriveGroupStatus(partitions)).toBe('critical');
+ });
+});
+
+// ── computeTargetInstances ───────────────────────────────────────────────────
+
+describe('computeTargetInstances', () => {
+ const base: ConsumerGroupScalingConfig = {
+ groupId: 'test-group',
+ scaleOutLagThreshold: 5_000,
+ scaleInLagThreshold: 200,
+ minInstances: 1,
+ maxInstances: 8,
+ currentInstances: 3,
+ };
+
+ it('returns null when lag is within stable range', () => {
+ expect(computeTargetInstances(1_000, base)).toBeNull();
+ expect(computeTargetInstances(200, base)).toBeNull();
+ });
+
+ it('scales out when lag exceeds scaleOutLagThreshold', () => {
+ const result = computeTargetInstances(10_000, base);
+ expect(result).not.toBeNull();
+ expect(result!).toBeGreaterThan(base.currentInstances);
+ expect(result!).toBeLessThanOrEqual(base.maxInstances);
+ });
+
+ it('scales in when lag drops below scaleInLagThreshold', () => {
+ const result = computeTargetInstances(50, base);
+ expect(result).not.toBeNull();
+ expect(result!).toBeLessThan(base.currentInstances);
+ expect(result!).toBeGreaterThanOrEqual(base.minInstances);
+ });
+
+ it('does not scale out beyond maxInstances', () => {
+ const atMax = { ...base, currentInstances: 8 };
+ expect(computeTargetInstances(100_000, atMax)).toBeNull();
+ });
+
+ it('does not scale in below minInstances', () => {
+ const atMin = { ...base, currentInstances: 1 };
+ expect(computeTargetInstances(0, atMin)).toBeNull();
+ });
+
+ it('scale-out factor grows with lag magnitude', () => {
+ const result1 = computeTargetInstances(5_000, base);
+ const result2 = computeTargetInstances(15_000, base);
+ // higher lag should produce a higher or equal target
+ expect((result2 ?? 0)).toBeGreaterThanOrEqual(result1 ?? 0);
+ });
+});
+
+// ── buildScalingEvent ────────────────────────────────────────────────────────
+
+describe('buildScalingEvent', () => {
+ it('produces a valid ScalingEvent', () => {
+ const event = buildScalingEvent('my-group', 2, 4, 8_000);
+ expect(event.groupId).toBe('my-group');
+ expect(event.previousInstances).toBe(2);
+ expect(event.targetInstances).toBe(4);
+ expect(event.lagAtTrigger).toBe(8_000);
+ expect(event.reason).toBe('lag-high');
+ expect(typeof event.id).toBe('string');
+ expect(event.id.length).toBeGreaterThan(0);
+ expect(typeof event.triggeredAt).toBe('number');
+ });
+
+ it('sets reason to lag-low when scaling in', () => {
+ const event = buildScalingEvent('my-group', 4, 2, 100);
+ expect(event.reason).toBe('lag-low');
+ });
+
+ it('sets reason to manual when specified', () => {
+ const event = buildScalingEvent('my-group', 3, 3, 0, 'manual');
+ expect(event.reason).toBe('manual');
+ });
+
+ it('generates unique IDs for distinct calls', () => {
+ const e1 = buildScalingEvent('g', 1, 2, 0);
+ const e2 = buildScalingEvent('g', 1, 2, 0);
+ expect(e1.id).not.toBe(e2.id);
+ });
+});
+
+// ── generateDemoSnapshot ─────────────────────────────────────────────────────
+
+describe('generateDemoSnapshot', () => {
+ it('returns an array of ConsumerGroupLag objects', () => {
+ const snapshot = generateDemoSnapshot();
+ expect(Array.isArray(snapshot)).toBe(true);
+ expect(snapshot.length).toBeGreaterThan(0);
+ });
+
+ it('each group has required fields', () => {
+ const snapshot = generateDemoSnapshot();
+ for (const group of snapshot) {
+ expect(typeof group.groupId).toBe('string');
+ expect(typeof group.topic).toBe('string');
+ expect(Array.isArray(group.partitions)).toBe(true);
+ expect(group.partitions.length).toBeGreaterThan(0);
+ expect(typeof group.totalLag).toBe('number');
+ expect(typeof group.maxPartitionLag).toBe('number');
+ expect(typeof group.capturedAt).toBe('number');
+ expect(['healthy', 'warning', 'critical']).toContain(group.status);
+ }
+ });
+
+ it('totalLag equals the sum of partition lags', () => {
+ const snapshot = generateDemoSnapshot();
+ for (const group of snapshot) {
+ const sum = group.partitions.reduce((s, p) => s + p.lag, 0);
+ expect(group.totalLag).toBe(sum);
+ }
+ });
+
+ it('maxPartitionLag equals the max of individual partition lags', () => {
+ const snapshot = generateDemoSnapshot();
+ for (const group of snapshot) {
+ const max = Math.max(...group.partitions.map((p) => p.lag));
+ expect(group.maxPartitionLag).toBe(max);
+ }
+ });
+
+ it('partition lag >= 0 for all partitions', () => {
+ const snapshot = generateDemoSnapshot();
+ for (const group of snapshot) {
+ for (const p of group.partitions) {
+ expect(p.lag).toBeGreaterThanOrEqual(0);
+ }
+ }
+ });
+});
+
+// ── generateDemoScalingStatuses ──────────────────────────────────────────────
+
+describe('generateDemoScalingStatuses', () => {
+ it('returns statuses for each demo group', () => {
+ const statuses = generateDemoScalingStatuses();
+ expect(statuses.length).toBeGreaterThan(0);
+ for (const s of statuses) {
+ expect(typeof s.groupId).toBe('string');
+ expect(typeof s.enabled).toBe('boolean');
+ expect(typeof s.config.minInstances).toBe('number');
+ expect(typeof s.config.maxInstances).toBe('number');
+ expect(s.config.minInstances).toBeGreaterThanOrEqual(1);
+ expect(s.config.maxInstances).toBeGreaterThanOrEqual(s.config.minInstances);
+ }
+ });
+});
+
+// ── createDemoKafkaMonitoringService ─────────────────────────────────────────
+
+describe('createDemoKafkaMonitoringService', () => {
+ it('fetchConsumerLag resolves to a non-empty array', async () => {
+ const svc = createDemoKafkaMonitoringService();
+ const result = await svc.fetchConsumerLag();
+ expect(Array.isArray(result)).toBe(true);
+ expect(result.length).toBeGreaterThan(0);
+ });
+
+ it('fetchScalingStatuses resolves to a non-empty array', async () => {
+ const svc = createDemoKafkaMonitoringService();
+ const result = await svc.fetchScalingStatuses();
+ expect(Array.isArray(result)).toBe(true);
+ expect(result.length).toBeGreaterThan(0);
+ });
+
+ it('triggerManualScale updates the instance count', async () => {
+ const svc = createDemoKafkaMonitoringService();
+ const statuses = await svc.fetchScalingStatuses();
+ const groupId = statuses[0].groupId;
+
+ const event = await svc.triggerManualScale(groupId, 5);
+ expect(event.groupId).toBe(groupId);
+ expect(event.targetInstances).toBe(5);
+ expect(event.reason).toBe('manual');
+
+ // Config should reflect the new instance count
+ const updated = await svc.fetchScalingStatuses();
+ const updatedStatus = updated.find((s) => s.groupId === groupId);
+ expect(updatedStatus?.config.currentInstances).toBe(5);
+ });
+});
+
+// ── Kafka store ──────────────────────────────────────────────────────────────
+
+describe('kafkaSlice store', () => {
+ // Dynamic import to get a fresh module-level store reference
+ it('upsertGroupLag and setGroups update the store correctly', async () => {
+ const { useKafkaStore } = await import('../store/kafkaSlice');
+ useKafkaStore.getState().reset();
+
+ const group = generateDemoSnapshot()[0];
+ useKafkaStore.getState().upsertGroupLag(group);
+
+ const state = useKafkaStore.getState();
+ expect(state.groups[group.groupId]).toBeDefined();
+ expect(state.groups[group.groupId].groupId).toBe(group.groupId);
+ });
+
+ it('addScalingEvent prepends to history and respects max 200', async () => {
+ const { useKafkaStore } = await import('../store/kafkaSlice');
+ useKafkaStore.getState().reset();
+
+ for (let i = 0; i < 205; i++) {
+ useKafkaStore.getState().addScalingEvent(
+ buildScalingEvent('g', 1, 2, 0),
+ );
+ }
+
+ expect(useKafkaStore.getState().scalingHistory.length).toBe(200);
+ });
+
+ it('markRefreshed sets isLoaded and lastRefreshedAt', async () => {
+ const { useKafkaStore } = await import('../store/kafkaSlice');
+ useKafkaStore.getState().reset();
+
+ const before = Date.now();
+ useKafkaStore.getState().markRefreshed();
+ const state = useKafkaStore.getState();
+
+ expect(state.isLoaded).toBe(true);
+ expect(state.lastRefreshedAt).toBeGreaterThanOrEqual(before);
+ expect(state.error).toBeNull();
+ });
+
+ it('setError records error message', async () => {
+ const { useKafkaStore } = await import('../store/kafkaSlice');
+ useKafkaStore.getState().reset();
+
+ useKafkaStore.getState().setError('something broke');
+ expect(useKafkaStore.getState().error).toBe('something broke');
+ });
+
+ it('reset returns store to initial state', async () => {
+ const { useKafkaStore } = await import('../store/kafkaSlice');
+ useKafkaStore.getState().markRefreshed();
+ useKafkaStore.getState().reset();
+
+ const state = useKafkaStore.getState();
+ expect(state.isLoaded).toBe(false);
+ expect(state.groups).toEqual({});
+ expect(state.scalingHistory).toEqual([]);
+ expect(state.error).toBeNull();
+ });
+});
+
+// ── Performance guard: computeTargetInstances P99 < 100ms ───────────────────
+
+describe('performance', () => {
+ it('computeTargetInstances completes 10 000 evaluations in < 100ms', () => {
+ const config: ConsumerGroupScalingConfig = {
+ groupId: 'perf-group',
+ scaleOutLagThreshold: 5_000,
+ scaleInLagThreshold: 200,
+ minInstances: 1,
+ maxInstances: 16,
+ currentInstances: 4,
+ };
+
+ const start = performance.now();
+ for (let i = 0; i < 10_000; i++) {
+ computeTargetInstances(i * 10, config);
+ }
+ const elapsed = performance.now() - start;
+ expect(elapsed).toBeLessThan(100);
+ });
+});
diff --git a/src/components/kafka/ConsumerGroupLagCard.tsx b/src/components/kafka/ConsumerGroupLagCard.tsx
new file mode 100644
index 0000000..99bc188
--- /dev/null
+++ b/src/components/kafka/ConsumerGroupLagCard.tsx
@@ -0,0 +1,196 @@
+'use client';
+
+// ConsumerGroupLagCard — per-group lag breakdown card.
+// Issue #109 — Kafka Consumer Lag Monitoring.
+
+import { useState } from 'react';
+import { LagStatusBadge } from './LagStatusBadge';
+import type { ConsumerGroupLag, ScalingStatus } from '../../types/kafka';
+
+interface ConsumerGroupLagCardProps {
+ group: ConsumerGroupLag;
+ scalingStatus?: ScalingStatus;
+ onManualScale?: (groupId: string, targetInstances: number) => void;
+}
+
+function formatLag(lag: number): string {
+ if (lag >= 1_000_000) return `${(lag / 1_000_000).toFixed(2)}M`;
+ if (lag >= 1_000) return `${(lag / 1_000).toFixed(1)}K`;
+ return String(lag);
+}
+
+function formatTime(ts: number): string {
+ return new Date(ts).toLocaleTimeString([], {
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ });
+}
+
+export function ConsumerGroupLagCard({
+ group,
+ scalingStatus,
+ onManualScale,
+}: ConsumerGroupLagCardProps) {
+ const [expanded, setExpanded] = useState(false);
+ const [scaleInput, setScaleInput] = useState('');
+
+ const { groupId, topic, partitions, totalLag, maxPartitionLag, capturedAt, status } = group;
+ const config = scalingStatus?.config;
+
+ function handleManualScale() {
+ const target = parseInt(scaleInput, 10);
+ if (!isNaN(target) && target > 0 && onManualScale) {
+ onManualScale(groupId, target);
+ setScaleInput('');
+ }
+ }
+
+ return (
+
+ {/* Header */}
+
+
+
+ Consumer Group
+
+
{groupId}
+
Topic: {topic}
+
+
+
+
+ {/* Summary stats */}
+
+
+
+
+ {config && (
+
+ )}
+
+
+ {/* Partition table toggle */}
+
+
+ {/* Partition breakdown */}
+ {expanded && (
+
+
+
+
+ | Partition |
+ Log-End Offset |
+ Consumer Offset |
+ Lag |
+ Status |
+
+
+
+ {partitions.map((p) => (
+
+ | {p.partition} |
+
+ {p.logEndOffset.toLocaleString()}
+ |
+
+ {p.consumerOffset.toLocaleString()}
+ |
+
+ {formatLag(p.lag)}
+ |
+
+
+ |
+
+ ))}
+
+
+
+ )}
+
+ {/* Manual scaling */}
+ {scalingStatus?.enabled && onManualScale && config && (
+
+
+ setScaleInput(e.target.value)}
+ placeholder={String(config.currentInstances)}
+ className="w-20 rounded-lg border border-white/10 bg-slate-800 px-2 py-1 text-xs text-white placeholder-slate-600 focus:outline-none focus:ring-2 focus:ring-amber-400"
+ aria-label={`Target instance count for ${groupId}`}
+ />
+
+
+ range {config.minInstances}–{config.maxInstances}
+
+
+ )}
+
+ {/* Last event */}
+ {scalingStatus?.lastEvent && (
+
+ Last scaling:{' '}
+
+ {scalingStatus.lastEvent.previousInstances} → {scalingStatus.lastEvent.targetInstances} instances
+ {' '}
+ at {formatTime(scalingStatus.lastEvent.triggeredAt)}
+
+ )}
+
+ Snapshot at {formatTime(capturedAt)}
+
+ );
+}
+
+function StatCell({
+ label,
+ value,
+ highlight = false,
+}: {
+ label: string;
+ value: string;
+ highlight?: boolean;
+}) {
+ return (
+
+
{label}
+
+ {value}
+
+
+ );
+}
diff --git a/src/components/kafka/KafkaMonitoringDashboard.tsx b/src/components/kafka/KafkaMonitoringDashboard.tsx
new file mode 100644
index 0000000..102ac2e
--- /dev/null
+++ b/src/components/kafka/KafkaMonitoringDashboard.tsx
@@ -0,0 +1,129 @@
+'use client';
+
+// KafkaMonitoringDashboard — top-level Kafka consumer lag & auto-scaling dashboard.
+// Issue #109 — system-wide implementation.
+//
+// Renders:
+// • A summary row (total groups, critical count, healthy count)
+// • A card per consumer group (ConsumerGroupLagCard)
+// • A scaling-event history table (ScalingHistoryTable)
+// • Live refresh indicator and manual refresh button
+
+import { useKafkaMonitoring } from '../../hooks/useKafkaMonitoring';
+import { ConsumerGroupLagCard } from './ConsumerGroupLagCard';
+import { ScalingHistoryTable } from './ScalingHistoryTable';
+import { LagStatusBadge } from './LagStatusBadge';
+
+function formatLastRefreshed(ts: number | null): string {
+ if (!ts) return 'Never';
+ return new Date(ts).toLocaleTimeString([], {
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ });
+}
+
+export function KafkaMonitoringDashboard() {
+ const {
+ groups,
+ scalingStatus,
+ scalingHistory,
+ isLoaded,
+ error,
+ lastRefreshedAt,
+ triggerManualScale,
+ refresh,
+ } = useKafkaMonitoring();
+
+ const groupList = Object.values(groups);
+ const criticalCount = groupList.filter((g) => g.status === 'critical').length;
+ const warningCount = groupList.filter((g) => g.status === 'warning').length;
+ const healthyCount = groupList.filter((g) => g.status === 'healthy').length;
+
+ return (
+
+ {/* Dashboard header */}
+
+
+
+ Kafka Consumer Lag Monitoring
+
+
+ Auto-scaling consumer groups — issue #109
+
+
+
+
+ {/* Summary pills */}
+ {groupList.length > 0 && (
+ <>
+ {criticalCount > 0 && (
+
+ )}
+ {warningCount > 0 && (
+
+ )}
+
+ >
+ )}
+
+ {/* Refresh controls */}
+
+ Last refresh: {formatLastRefreshed(lastRefreshedAt)}
+
+
+
+
+
+ {/* Error banner */}
+ {error && (
+
+ {error}
+
+ )}
+
+ {/* Loading state */}
+ {!isLoaded && !error && (
+
+
+ Loading consumer group data…
+
+ )}
+
+ {/* Group cards */}
+ {isLoaded && groupList.length === 0 && !error && (
+
+ No consumer groups found.
+
+ )}
+
+ {isLoaded && groupList.length > 0 && (
+
+ {groupList.map((group) => (
+
+ ))}
+
+ )}
+
+ {/* Scaling history */}
+ {isLoaded && (
+
+ )}
+
+ );
+}
diff --git a/src/components/kafka/LagStatusBadge.tsx b/src/components/kafka/LagStatusBadge.tsx
new file mode 100644
index 0000000..f8a38a8
--- /dev/null
+++ b/src/components/kafka/LagStatusBadge.tsx
@@ -0,0 +1,38 @@
+'use client';
+
+// LagStatusBadge — colour-coded pill indicating consumer group / partition health.
+// Issue #109 — Kafka Consumer Lag Monitoring.
+
+import type { PartitionLagStatus } from '../../types/kafka';
+
+interface LagStatusBadgeProps {
+ status: PartitionLagStatus;
+ /** Accessible label override. Defaults to the status string. */
+ label?: string;
+}
+
+const STATUS_STYLES: Record = {
+ healthy: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30',
+ warning: 'bg-amber-400/15 text-amber-400 border-amber-400/30',
+ critical: 'bg-rose-500/15 text-rose-400 border-rose-500/30',
+};
+
+const STATUS_DOT: Record = {
+ healthy: 'bg-emerald-400',
+ warning: 'bg-amber-400 animate-pulse',
+ critical: 'bg-rose-500 animate-pulse',
+};
+
+export function LagStatusBadge({ status, label }: LagStatusBadgeProps) {
+ const displayLabel = label ?? status.charAt(0).toUpperCase() + status.slice(1);
+ return (
+
+
+ {displayLabel}
+
+ );
+}
diff --git a/src/components/kafka/ScalingHistoryTable.tsx b/src/components/kafka/ScalingHistoryTable.tsx
new file mode 100644
index 0000000..fda6150
--- /dev/null
+++ b/src/components/kafka/ScalingHistoryTable.tsx
@@ -0,0 +1,94 @@
+'use client';
+
+// ScalingHistoryTable — compact table of recent auto-scaling events.
+// Issue #109 — Kafka Consumer Lag Monitoring.
+
+import type { ScalingEvent } from '../../types/kafka';
+
+interface ScalingHistoryTableProps {
+ events: ScalingEvent[];
+ /** Maximum rows to display. Defaults to 20. */
+ maxRows?: number;
+}
+
+const REASON_LABEL: Record = {
+ 'lag-high': '↑ Lag high',
+ 'lag-low': '↓ Lag low',
+ manual: '⚙ Manual',
+};
+
+const REASON_COLOR: Record = {
+ 'lag-high': 'text-rose-400',
+ 'lag-low': 'text-emerald-400',
+ manual: 'text-amber-400',
+};
+
+function formatTs(ts: number): string {
+ return new Date(ts).toLocaleString([], {
+ month: 'short',
+ day: 'numeric',
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ });
+}
+
+function formatLag(lag: number): string {
+ if (lag >= 1_000_000) return `${(lag / 1_000_000).toFixed(2)}M`;
+ if (lag >= 1_000) return `${(lag / 1_000).toFixed(1)}K`;
+ return String(lag);
+}
+
+export function ScalingHistoryTable({
+ events,
+ maxRows = 20,
+}: ScalingHistoryTableProps) {
+ const visible = events.slice(0, maxRows);
+
+ return (
+
+ Scaling History
+
+ {visible.length === 0 ? (
+
+ No scaling events recorded yet.
+
+ ) : (
+
+
+
+
+ | Time |
+ Group |
+ Instances |
+ Lag at Trigger |
+ Reason |
+
+
+
+ {visible.map((ev) => (
+
+ | {formatTs(ev.triggeredAt)} |
+ {ev.groupId} |
+
+ {ev.previousInstances} → {ev.targetInstances}
+ |
+ {formatLag(ev.lagAtTrigger)} |
+
+ {REASON_LABEL[ev.reason]}
+ |
+
+ ))}
+
+
+
+ )}
+
+ );
+}
diff --git a/src/hooks/useKafkaMonitoring.ts b/src/hooks/useKafkaMonitoring.ts
new file mode 100644
index 0000000..0fafe51
--- /dev/null
+++ b/src/hooks/useKafkaMonitoring.ts
@@ -0,0 +1,155 @@
+'use client';
+
+// useKafkaMonitoring — Kafka consumer lag polling and auto-scaling hook.
+// Issue #109 — system-wide implementation.
+//
+// Polls the Kafka monitoring service on a configurable interval, writes
+// snapshots into the Zustand kafkaSlice, and evaluates auto-scaling decisions
+// locally so the UI reflects the latest state without a round-trip.
+//
+// Performance target: < 100ms P99 for the synchronous evaluation path.
+
+import { useEffect, useRef, useCallback } from 'react';
+import { useKafkaStore } from '../store/kafkaSlice';
+import {
+ createDemoKafkaMonitoringService,
+ computeTargetInstances,
+ buildScalingEvent,
+} from '../services/kafkaMonitoringService';
+import type { KafkaMonitoringProvider } from '../services/kafkaMonitoringService';
+
+/** Default poll interval in milliseconds. */
+const DEFAULT_POLL_INTERVAL_MS = 15_000;
+
+export interface UseKafkaMonitoringOptions {
+ /** Set to false to pause polling. Defaults to true. */
+ enabled?: boolean;
+ /** Poll interval in ms. Defaults to 15 000. */
+ pollIntervalMs?: number;
+ /**
+ * Optional custom provider. Falls back to the demo service when omitted,
+ * matching the project-wide pattern of demo-first development.
+ */
+ provider?: KafkaMonitoringProvider;
+}
+
+export function useKafkaMonitoring(options: UseKafkaMonitoringOptions = {}) {
+ const {
+ enabled = true,
+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+ provider,
+ } = options;
+
+ const store = useKafkaStore();
+ const serviceRef = useRef(
+ provider ?? createDemoKafkaMonitoringService(),
+ );
+ const timerRef = useRef | undefined>(undefined);
+ const mountedRef = useRef(true);
+
+ const poll = useCallback(async () => {
+ const svc = serviceRef.current;
+
+ try {
+ // Fetch lag snapshots and scaling statuses in parallel.
+ const [groups, statuses] = await Promise.all([
+ svc.fetchConsumerLag(),
+ svc.fetchScalingStatuses(),
+ ]);
+
+ if (!mountedRef.current) return;
+
+ store.setGroups(groups);
+ statuses.forEach((s) => store.upsertScalingStatus(s));
+ store.markRefreshed();
+
+ // Evaluate auto-scaling decisions for each group.
+ for (const status of statuses) {
+ if (!status.enabled) continue;
+
+ const groupLag = groups.find((g) => g.groupId === status.groupId);
+ if (!groupLag) continue;
+
+ const target = computeTargetInstances(groupLag.totalLag, status.config);
+ if (target === null) continue;
+
+ const event = buildScalingEvent(
+ status.groupId,
+ status.config.currentInstances,
+ target,
+ groupLag.totalLag,
+ );
+
+ // Persist to store; production would also call svc.triggerManualScale().
+ store.addScalingEvent(event);
+ store.upsertScalingStatus({
+ ...status,
+ config: { ...status.config, currentInstances: target },
+ lastEvent: event,
+ });
+ }
+ } catch (err) {
+ if (!mountedRef.current) return;
+ store.setError(err instanceof Error ? err.message : 'Kafka poll failed');
+ }
+ }, [store]);
+
+ // Trigger an immediate manual scale action.
+ const triggerManualScale = useCallback(
+ async (groupId: string, targetInstances: number) => {
+ try {
+ const event = await serviceRef.current.triggerManualScale(groupId, targetInstances);
+ store.addScalingEvent(event);
+
+ const existing = useKafkaStore.getState().scalingStatus[groupId];
+ if (existing) {
+ store.upsertScalingStatus({
+ ...existing,
+ config: { ...existing.config, currentInstances: targetInstances },
+ lastEvent: event,
+ });
+ }
+ } catch (err) {
+ store.setError(
+ err instanceof Error ? err.message : 'Manual scale request failed',
+ );
+ }
+ },
+ [store],
+ );
+
+ useEffect(() => {
+ if (!enabled) return;
+
+ mountedRef.current = true;
+
+ // Immediate first poll, then schedule repeating interval.
+ void poll();
+
+ function scheduleNext() {
+ timerRef.current = setTimeout(async () => {
+ await poll();
+ if (mountedRef.current) scheduleNext();
+ }, pollIntervalMs);
+ }
+ scheduleNext();
+
+ return () => {
+ mountedRef.current = false;
+ if (timerRef.current !== undefined) {
+ clearTimeout(timerRef.current);
+ }
+ };
+ }, [enabled, pollIntervalMs, poll]);
+
+ return {
+ groups: store.groups,
+ scalingStatus: store.scalingStatus,
+ scalingHistory: store.scalingHistory,
+ isLoaded: store.isLoaded,
+ error: store.error,
+ lastRefreshedAt: store.lastRefreshedAt,
+ triggerManualScale,
+ refresh: poll,
+ };
+}
diff --git a/src/services/kafkaMonitoringService.ts b/src/services/kafkaMonitoringService.ts
new file mode 100644
index 0000000..da52982
--- /dev/null
+++ b/src/services/kafkaMonitoringService.ts
@@ -0,0 +1,250 @@
+// Kafka consumer lag monitoring service.
+// Issue #109 — system-wide implementation.
+//
+// In production this module would call a real Kafka metrics endpoint (e.g.
+// Burrow, Confluent REST Proxy, or a bespoke scraper). In demo mode it returns
+// deterministic synthetic data so the UI can be developed and tested without
+// a live Kafka cluster.
+
+import type {
+ ConsumerGroupLag,
+ ConsumerGroupScalingConfig,
+ PartitionLag,
+ PartitionLagStatus,
+ ScalingEvent,
+ ScalingStatus,
+} from '../types/kafka';
+
+// ── Lag status thresholds ────────────────────────────────────────────────────
+
+/** Per-partition lag that triggers a "warning" status. */
+const WARNING_LAG_THRESHOLD = 1_000;
+/** Per-partition lag that triggers a "critical" status. */
+const CRITICAL_LAG_THRESHOLD = 10_000;
+
+export function derivePartitionStatus(lag: number): PartitionLagStatus {
+ if (lag >= CRITICAL_LAG_THRESHOLD) return 'critical';
+ if (lag >= WARNING_LAG_THRESHOLD) return 'warning';
+ return 'healthy';
+}
+
+export function deriveGroupStatus(partitions: PartitionLag[]): PartitionLagStatus {
+ if (partitions.some((p) => p.status === 'critical')) return 'critical';
+ if (partitions.some((p) => p.status === 'warning')) return 'warning';
+ return 'healthy';
+}
+
+// ── Auto-scaling logic ────────────────────────────────────────────────────────
+
+/**
+ * Given current lag and a scaling config, compute the target instance count.
+ * Returns `null` when no change is required.
+ *
+ * Performance target: pure sync function, always < 100 µs.
+ */
+export function computeTargetInstances(
+ totalLag: number,
+ config: ConsumerGroupScalingConfig,
+): number | null {
+ const { currentInstances, minInstances, maxInstances } = config;
+
+ if (totalLag >= config.scaleOutLagThreshold && currentInstances < maxInstances) {
+ // Scale out: add one instance per 10× lag above threshold, capped at max.
+ const factor = Math.ceil(totalLag / config.scaleOutLagThreshold);
+ return Math.min(currentInstances + factor, maxInstances);
+ }
+
+ if (totalLag < config.scaleInLagThreshold && currentInstances > minInstances) {
+ // Scale in: remove one instance.
+ return Math.max(currentInstances - 1, minInstances);
+ }
+
+ return null; // no change
+}
+
+/**
+ * Build a ScalingEvent from a scaling decision.
+ */
+export function buildScalingEvent(
+ groupId: string,
+ previousInstances: number,
+ targetInstances: number,
+ lagAtTrigger: number,
+ reason: ScalingEvent['reason'] = 'lag-high',
+): ScalingEvent {
+ let derivedReason: ScalingEvent['reason'];
+ if (reason === 'manual') {
+ derivedReason = 'manual';
+ } else {
+ derivedReason = targetInstances > previousInstances ? 'lag-high' : 'lag-low';
+ }
+
+ return {
+ id: `${groupId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+ groupId,
+ triggeredAt: Date.now(),
+ previousInstances,
+ targetInstances,
+ reason: derivedReason,
+ lagAtTrigger,
+ };
+}
+
+// ── Demo data generation ──────────────────────────────────────────────────────
+
+/** Lightweight LCG for reproducible-ish demo data that evolves each call. */
+let _seed = 0xdeadbeef;
+function lcg(): number {
+ _seed = Math.imul(_seed ^ (_seed >>> 13), 0x9e3779b9);
+ _seed = _seed ^ (_seed >>> 11);
+ return (_seed >>> 0) / 0x1_0000_0000;
+}
+
+const DEMO_TOPICS = [
+ 'validator-attestations',
+ 'block-proposals',
+ 'sync-committee-duties',
+ 'reward-calculations',
+ 'node-health-events',
+];
+
+const DEMO_GROUPS = [
+ 'attestation-processor',
+ 'block-builder',
+ 'sync-committee-monitor',
+ 'reward-indexer',
+ 'health-aggregator',
+];
+
+const DEMO_SCALING_CONFIGS: ConsumerGroupScalingConfig[] = DEMO_GROUPS.map((groupId, i) => ({
+ groupId,
+ scaleOutLagThreshold: 5_000,
+ scaleInLagThreshold: 200,
+ minInstances: 1,
+ maxInstances: 8,
+ currentInstances: 2 + (i % 3),
+}));
+
+/** Generate a deterministic-ish snapshot for all demo consumer groups. */
+export function generateDemoSnapshot(): ConsumerGroupLag[] {
+ return DEMO_GROUPS.map((groupId, gi) => {
+ const topic = DEMO_TOPICS[gi % DEMO_TOPICS.length];
+ const partitionCount = 4 + (gi % 4); // 4–7 partitions per group
+ const partitions: PartitionLag[] = Array.from({ length: partitionCount }, (_, pi) => {
+ const logEndOffset = 100_000 + Math.floor(lcg() * 50_000);
+ // Occasionally spike one partition for demo variety
+ const lagBase = gi === 2 && pi === 1 ? 12_000 : gi === 0 ? 800 : 200;
+ const lag = Math.max(0, Math.floor(lagBase + lcg() * lagBase));
+ const consumerOffset = logEndOffset - lag;
+ const status = derivePartitionStatus(lag);
+ return { partition: pi, logEndOffset, consumerOffset, lag, status };
+ });
+
+ const totalLag = partitions.reduce((s, p) => s + p.lag, 0);
+ const maxPartitionLag = Math.max(...partitions.map((p) => p.lag));
+
+ return {
+ groupId,
+ topic,
+ partitions,
+ totalLag,
+ maxPartitionLag,
+ capturedAt: Date.now(),
+ status: deriveGroupStatus(partitions),
+ };
+ });
+}
+
+/** Generate initial scaling status for all demo groups. */
+export function generateDemoScalingStatuses(): ScalingStatus[] {
+ return DEMO_SCALING_CONFIGS.map((config) => ({
+ groupId: config.groupId,
+ enabled: true,
+ config,
+ lastEvent: null,
+ }));
+}
+
+// ── HTTP service ──────────────────────────────────────────────────────────────
+
+export interface KafkaMonitoringProvider {
+ /**
+ * Fetch the current consumer lag snapshot for all tracked groups.
+ * Returns an empty array on any error.
+ */
+ fetchConsumerLag(): Promise;
+
+ /**
+ * Fetch current scaling status for all groups.
+ */
+ fetchScalingStatuses(): Promise;
+
+ /**
+ * Trigger a manual scaling action for a group.
+ * Returns the resulting ScalingEvent.
+ */
+ triggerManualScale(groupId: string, targetInstances: number): Promise;
+}
+
+/** Demo provider — deterministic, zero network calls. */
+export function createDemoKafkaMonitoringService(): KafkaMonitoringProvider {
+ const scalingStatuses = generateDemoScalingStatuses();
+
+ return {
+ async fetchConsumerLag() {
+ return generateDemoSnapshot();
+ },
+
+ async fetchScalingStatuses() {
+ return scalingStatuses;
+ },
+
+ async triggerManualScale(groupId, targetInstances) {
+ const status = scalingStatuses.find((s) => s.groupId === groupId);
+ const previous = status?.config.currentInstances ?? 1;
+ const event = buildScalingEvent(groupId, previous, targetInstances, 0, 'manual');
+ if (status) {
+ status.config.currentInstances = targetInstances;
+ status.lastEvent = event;
+ }
+ return event;
+ },
+ };
+}
+
+/** Production provider — calls real Kafka metrics endpoints. */
+export function createKafkaMonitoringService(baseUrl: string): KafkaMonitoringProvider {
+ return {
+ async fetchConsumerLag() {
+ try {
+ const res = await fetch(`${baseUrl}/api/kafka/consumer-lag`);
+ if (!res.ok) return [];
+ return (await res.json()) as ConsumerGroupLag[];
+ } catch {
+ return [];
+ }
+ },
+
+ async fetchScalingStatuses() {
+ try {
+ const res = await fetch(`${baseUrl}/api/kafka/scaling-status`);
+ if (!res.ok) return [];
+ return (await res.json()) as ScalingStatus[];
+ } catch {
+ return [];
+ }
+ },
+
+ async triggerManualScale(groupId, targetInstances) {
+ const res = await fetch(`${baseUrl}/api/kafka/scale`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ groupId, targetInstances }),
+ });
+ if (!res.ok) {
+ throw new Error(`Scale request failed: ${res.status}`);
+ }
+ return (await res.json()) as ScalingEvent;
+ },
+ };
+}
diff --git a/src/store/kafkaSlice.ts b/src/store/kafkaSlice.ts
new file mode 100644
index 0000000..22f4336
--- /dev/null
+++ b/src/store/kafkaSlice.ts
@@ -0,0 +1,71 @@
+// Kafka consumer lag monitoring and auto-scaling store.
+// Issue #109 — system-wide implementation.
+
+import { create } from 'zustand';
+import type {
+ ConsumerGroupLag,
+ KafkaMonitoringState,
+ ScalingEvent,
+ ScalingStatus,
+} from '../types/kafka';
+
+interface KafkaMonitoringActions {
+ /** Replace (or insert) a group lag snapshot. */
+ upsertGroupLag: (lag: ConsumerGroupLag) => void;
+ /** Replace all group lag snapshots at once. */
+ setGroups: (groups: ConsumerGroupLag[]) => void;
+ /** Update scaling status for a group. */
+ upsertScalingStatus: (status: ScalingStatus) => void;
+ /** Prepend a new scaling event to the history ring-buffer (max 200). */
+ addScalingEvent: (event: ScalingEvent) => void;
+ /** Mark store as loaded and record the refresh timestamp. */
+ markRefreshed: () => void;
+ /** Set the current error message. Pass null to clear. */
+ setError: (error: string | null) => void;
+ /** Reset the store to initial state. */
+ reset: () => void;
+}
+
+const MAX_SCALING_HISTORY = 200;
+
+const initialState: KafkaMonitoringState = {
+ groups: {},
+ scalingStatus: {},
+ scalingHistory: [],
+ isLoaded: false,
+ error: null,
+ lastRefreshedAt: null,
+};
+
+export const useKafkaStore = create(
+ (set) => ({
+ ...initialState,
+
+ upsertGroupLag: (lag) =>
+ set((s) => ({
+ groups: { ...s.groups, [lag.groupId]: lag },
+ })),
+
+ setGroups: (groups) =>
+ set(() => ({
+ groups: Object.fromEntries(groups.map((g) => [g.groupId, g])),
+ })),
+
+ upsertScalingStatus: (status) =>
+ set((s) => ({
+ scalingStatus: { ...s.scalingStatus, [status.groupId]: status },
+ })),
+
+ addScalingEvent: (event) =>
+ set((s) => ({
+ scalingHistory: [event, ...s.scalingHistory].slice(0, MAX_SCALING_HISTORY),
+ })),
+
+ markRefreshed: () =>
+ set({ isLoaded: true, lastRefreshedAt: Date.now(), error: null }),
+
+ setError: (error) => set({ error }),
+
+ reset: () => set(initialState),
+ }),
+);
diff --git a/src/types/kafka.ts b/src/types/kafka.ts
new file mode 100644
index 0000000..bf118d2
--- /dev/null
+++ b/src/types/kafka.ts
@@ -0,0 +1,96 @@
+// Kafka consumer lag monitoring and auto-scaling types.
+// Issue #109 — system-wide implementation.
+
+/** Health status of a single consumer group partition. */
+export type PartitionLagStatus = 'healthy' | 'warning' | 'critical';
+
+/** A single partition's lag snapshot. */
+export interface PartitionLag {
+ /** Kafka partition number (0-indexed). */
+ partition: number;
+ /** Log-end offset — latest offset produced to this partition. */
+ logEndOffset: number;
+ /** Current offset committed by the consumer group. */
+ consumerOffset: number;
+ /** Lag = logEndOffset − consumerOffset. */
+ lag: number;
+ /** Derived status based on lag thresholds. */
+ status: PartitionLagStatus;
+}
+
+/** Aggregated lag snapshot for one consumer group across all partitions. */
+export interface ConsumerGroupLag {
+ /** Unique consumer group ID. */
+ groupId: string;
+ /** Kafka topic being consumed. */
+ topic: string;
+ /** Per-partition breakdowns. */
+ partitions: PartitionLag[];
+ /** Sum of lag across all partitions. */
+ totalLag: number;
+ /** Max single-partition lag. */
+ maxPartitionLag: number;
+ /** Timestamp (ms) when this snapshot was taken. */
+ capturedAt: number;
+ /** Derived group-level health. */
+ status: PartitionLagStatus;
+}
+
+/** Auto-scaling decision record for a consumer group. */
+export interface ScalingEvent {
+ /** Unique event ID. */
+ id: string;
+ /** Consumer group this event applies to. */
+ groupId: string;
+ /** UTC timestamp (ms) of the scaling action. */
+ triggeredAt: number;
+ /** Consumer instance count before the action. */
+ previousInstances: number;
+ /** Consumer instance count after the action. */
+ targetInstances: number;
+ /** Reason code for the decision. */
+ reason: 'lag-high' | 'lag-low' | 'manual';
+ /** Total lag that triggered the event. */
+ lagAtTrigger: number;
+}
+
+/** Configuration thresholds for auto-scaling a consumer group. */
+export interface ConsumerGroupScalingConfig {
+ groupId: string;
+ /** Lag above this value triggers a scale-out. */
+ scaleOutLagThreshold: number;
+ /** Lag below this value triggers a scale-in. */
+ scaleInLagThreshold: number;
+ /** Minimum number of consumer instances. */
+ minInstances: number;
+ /** Maximum number of consumer instances. */
+ maxInstances: number;
+ /** Current running instance count. */
+ currentInstances: number;
+}
+
+/** Current auto-scaling status for a consumer group. */
+export interface ScalingStatus {
+ groupId: string;
+ /** Whether auto-scaling is enabled for this group. */
+ enabled: boolean;
+ config: ConsumerGroupScalingConfig;
+ /** Most recent scaling event, if any. */
+ lastEvent: ScalingEvent | null;
+}
+
+/** Zustand store state shape for the Kafka monitoring slice. */
+export interface KafkaMonitoringState {
+ /** All tracked consumer group lag snapshots, keyed by groupId. */
+ groups: Record;
+ /** Scaling status keyed by groupId. */
+ scalingStatus: Record;
+ /** History of scaling events (most-recent first). */
+ scalingHistory: ScalingEvent[];
+ /** Whether initial data has been loaded. */
+ isLoaded: boolean;
+ /** Last poll error message, if any. */
+ error: string | null;
+ /** Timestamp (ms) of the last successful refresh. */
+ lastRefreshedAt: number | null;
+}