feat(actors): expose container system metrics

Author: NathanFlurry

docker/dev-full/docker-compose.yml

@@ -233,6 +233,8 @@ services:
       # We only reserve 100 ports instead of the default 22,000. See
       # rivet-guard for explanation.
       - "7600-7699:7600-7699"
+      # cAdvisor metrics endpoint
+      - "7780:7780"
     networks:
       - rivet-network
 

docker/dev-full/otel-collector/config.yaml

@@ -5,6 +5,14 @@ receivers:
         endpoint: 0.0.0.0:4317
       http:
         endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'cadvisor'
+          static_configs:
+            - targets: ['rivet-client:7780']
+          metrics_path: /metrics
+          scrape_interval: 30s
 
 processors:
   batch:
@@ -52,7 +60,7 @@ service:
       processors: [batch]
       exporters: [clickhouse]
     metrics:
-      receivers: [otlp]
+      receivers: [otlp, prometheus]
       processors: [batch]
       exporters: [clickhouse]
 

docker/universal/Dockerfile

@@ -120,8 +120,8 @@ RUN apt-get update -y && \
     apt-get install -y ca-certificates openssl curl tini curl && \
     curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.1.60/libfdb_c.x86_64.so"
 
-# MARK: Runner (Full)
-FROM --platform=linux/amd64 base-runner AS client-full
+# MARK: Runner (Slim)
+FROM --platform=linux/amd64 base-runner AS client-slim
 ARG CNI_PLUGINS_VERSION=1.3.0
 RUN apt-get install -y skopeo iproute2 runc dnsutils && \
 	echo "Downloading lz4" && \
@@ -142,6 +142,21 @@ COPY ./docker/dev-full/rivet-client/rivet-actor.conflist /opt/cni/config/rivet-a
 COPY --from=builder /app/dist/rivet-client /app/dist/rivet-container-runner /usr/local/bin/
 ENTRYPOINT ["/usr/bin/tini", "--", "entrypoint.sh"]
 
+# MARK: Runner (Full)
+FROM client-slim AS client-full
+ARG CADVISOR_VERSION=v0.52.0
+RUN apt-get update -y && \
+    apt-get install -y wget && \
+    wget -O /usr/local/bin/cadvisor "https://github.com/google/cadvisor/releases/download/${CADVISOR_VERSION}/cadvisor-${CADVISOR_VERSION}-linux-amd64" && \
+    chmod +x /usr/local/bin/cadvisor && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY docker/universal/client-full-entrypoint.sh /usr/local/bin/client-full-entrypoint.sh
+RUN chmod +x /usr/local/bin/client-full-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/client-full-entrypoint.sh"]
+
 # MARK: Monlith
 FROM --platform=linux/amd64 debian:12.9-slim AS monolith
 ENV DEBIAN_FRONTEND=noninteractive

docker/universal/client-full-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# Start cadvisor in the background
+cadvisor \
+  --port=7780 \
+  --listen_ip=0.0.0.0 \
+  --prometheus_endpoint="/metrics" \
+  --enable_metrics=cpu,cpuLoad,memory,network,disk,diskIO,oom_event,process,tcp,udp \
+  --docker_only=false \
+  --disable_root_cgroup_stats=false &
+
+# TODO:
+# --raw_cgroup_prefix_whitelist="" \
+
+# Start rivet-client with all passed arguments
+exec rivet-client "$@"

examples/system-test-actor/src/shared/server.ts

@@ -7,14 +7,14 @@ export function createAndStartServer(
 	getUpgradeWebSocket: GetUpgradeWebSocketFn,
 ): { app: Hono; port: number } {
 	// Setup auto-exit timer
-	setTimeout(() => {
-		console.error(
-			"Actor should've been destroyed by now. Automatically exiting.",
-		);
-
-		if (typeof Deno !== "undefined") Deno.exit(1);
-		else process.exit(1);
-	}, 60 * 1000);
+	// setTimeout(() => {
+	// 	console.error(
+	// 		"Actor should've been destroyed by now. Automatically exiting.",
+	// 	);
+	//
+	// 	if (typeof Deno !== "undefined") Deno.exit(1);
+	// 	else process.exit(1);
+	// }, 60 * 1000);
 
 	let tickIndex = 0;
 	setInterval(() => {

frontend/apps/hub/src/domains/project/components/actors/actors-provider.tsx

@@ -12,6 +12,7 @@ import {
 	actorBuildsAtom,
 	createActorAtom,
 	type Logs,
+	type Metrics,
 	actorsQueryAtom,
 	actorsInternalFilterAtom,
 	type Actor,
@@ -30,6 +31,7 @@ import {
 	createActorEndpoint,
 	destroyActorMutationOptions,
 	actorLogsQueryOptions,
+	actorMetricsQueryOptions,
 	actorRegionsQueryOptions,
 	actorBuildsQueryOptions,
 } from "../../queries";
@@ -257,9 +259,63 @@ export function ActorsProvider({
 								};
 							};
 
+							const metrics = atom({
+								metrics: { cpu: null, memory: null } as Metrics,
+								status: "pending",
+							});
+							metrics.onMount = (set) => {
+								const metricsObserver = new QueryObserver(
+									queryClient,
+									actorMetricsQueryOptions({
+										projectNameId,
+										environmentNameId,
+										actorId: actor.id,
+									}, { refetchInterval: 5000 }),
+								);
+
+								type MetricsQuery = {
+									status: string;
+									data?: Awaited<
+										ReturnType<
+											Exclude<
+												ReturnType<
+													typeof actorMetricsQueryOptions
+												>["queryFn"],
+												undefined
+											>
+										>
+									>;
+								};
+
+								function updateMetrics(query: MetricsQuery) {
+									const data = query.data;
+									set((prev) => ({
+										...prev,
+										...data,
+										status: query.status,
+									}));
+								}
+
+								const subMetrics = metricsObserver.subscribe(
+									(query) => {
+										updateMetrics(query);
+									},
+								);
+
+								updateMetrics(
+									metricsObserver.getCurrentQuery().state,
+								);
+
+								return () => {
+									metricsObserver.destroy();
+									subMetrics();
+								};
+							};
+
 							return {
 								...actor,
 								logs,
+								metrics,
 								destroy,
 								status: getActorStatus(actor),
 							};

frontend/apps/hub/src/domains/project/queries/actors/query-options.ts

@@ -255,6 +255,73 @@ export const actorLogsQueryOptions = (
 	});
 };
 
+export const actorMetricsQueryOptions = (
+	{
+		projectNameId,
+		environmentNameId,
+		actorId,
+	}: {
+		projectNameId: string;
+		environmentNameId: string;
+		actorId: string;
+	},
+	opts: { refetchInterval?: number } = {},
+) => {
+	return queryOptions({
+		...opts,
+		queryKey: [
+			"project",
+			projectNameId,
+			"environment",
+			environmentNameId,
+			"actor",
+			actorId,
+			"metrics",
+		] as const,
+		queryFn: async ({
+			signal: abortSignal,
+			queryKey: [, project, , environment, , actorId],
+		}) => {
+			const now = Date.now();
+			const start = now - 60000; // Last minute
+			const end = now;
+			
+			const response = await rivetClient.actors.metrics.get(
+				{
+					project,
+					environment,
+					start,
+					end,
+					interval: 10000, // 10 second intervals
+					actorIdsJson: JSON.stringify([actorId]),
+					metricsJson: JSON.stringify([
+						"container_cpu_usage_seconds_total",
+						"container_memory_usage_bytes"
+					]),
+				},
+				{ abortSignal },
+			);
+
+			// Extract the latest metrics values
+			let cpu: number | null = null;
+			let memory: number | null = null;
+
+			if (response.metrics && response.metrics.length >= 2) {
+				const cpuData = response.metrics[0];
+				const memoryData = response.metrics[1];
+				
+				// Get the last data point for each metric
+				cpu = cpuData && cpuData.length > 0 ? cpuData[cpuData.length - 1] : null;
+				memory = memoryData && memoryData.length > 0 ? memoryData[memoryData.length - 1] : null;
+			}
+
+			return {
+				metrics: { cpu, memory },
+			};
+		},
+	});
+};
+
 export const actorBuildsQueryOptions = ({
 	projectNameId,
 	environmentNameId,

frontend/packages/components/src/actors/actor-config-tab.tsx

@@ -1,5 +1,6 @@
 import { Button, DocsSheet, ScrollArea } from "@rivet-gg/components";
 import { Icon, faBooks } from "@rivet-gg/icons";
+import { ActorMetrics } from "./actor-metrics";
 import { ActorGeneral } from "./actor-general";
 import { ActorNetwork } from "./actor-network";
 import { ActorRuntime } from "./actor-runtime";
@@ -23,6 +24,7 @@ export function ActorConfigTab(props: ActorConfigTabProps) {
 					</Button>
 				</DocsSheet>
 			</div>
+			<ActorMetrics {...props} />
 			<ActorGeneral {...props} />
 			<ActorNetwork {...props} />
 			<ActorRuntime {...props} />

frontend/packages/components/src/actors/actor-context.tsx

@@ -25,6 +25,7 @@ export type Actor = Omit<
 	lifecycle?: Rivet.actor.Lifecycle;
 	endpoint?: string;
 	logs: LogsAtom;
+	metrics: MetricsAtom;
 	network?: Rivet.actor.Network | null;
 	resources?: Rivet.actor.Resources | null;
 	runtime?: Rivet.actor.Runtime | null;
@@ -43,6 +44,11 @@ export type Logs = {
 	properties: Record<string, unknown>;
 }[];
 
+export type Metrics = {
+	cpu: number | null;
+	memory: number | null;
+};
+
 export type Build = Rivet.actor.Build;
 export type DestroyActor = {
 	isDestroying: boolean;
@@ -55,6 +61,11 @@ export type LogsAtom = Atom<{
 	// query status
 	status: string;
 }>;
+export type MetricsAtom = Atom<{
+	metrics: Metrics;
+	// query status
+	status: string;
+}>;
 export type BuildAtom = Atom<Build>;
 export type DestroyActorAtom = Atom<DestroyActor>;
 

frontend/packages/components/src/actors/actor-metrics.tsx

@@ -0,0 +1,49 @@
+import { useAtomValue } from "jotai";
+import { selectAtom } from "jotai/utils";
+import equal from "fast-deep-equal";
+import { Dd, Dl, Dt, Flex } from "@rivet-gg/components";
+import type { Actor, ActorAtom } from "./actor-context";
+
+const selector = (a: Actor) => ({
+	metrics: a.metrics,
+});
+
+export interface ActorMetricsProps {
+	actor: ActorAtom;
+}
+
+export function ActorMetrics({ actor }: ActorMetricsProps) {
+	const { metrics } = useAtomValue(selectAtom(actor, selector, equal));
+	const metricsData = useAtomValue(metrics);
+
+	const formatCpuUsage = (cpu: number | null) => {
+		if (cpu === null) return "n/a";
+		return `${(cpu * 100).toFixed(2)}%`;
+	};
+
+	const formatMemoryUsage = (memory: number | null) => {
+		if (memory === null) return "n/a";
+		return `${(memory / 1024 / 1024).toFixed(1)} MB`;
+	};
+
+	const isLoading = metricsData.status === "pending";
+	const hasError = metricsData.status === "error";
+
+	return (
+		<div className="px-4 my-8">
+			<h3 className="mb-2 font-semibold">Metrics</h3>
+			<Flex gap="2" direction="col" className="text-xs">
+				<Dl>
+					<Dt>CPU Usage</Dt>
+					<Dd className={hasError ? "text-destructive" : ""}>
+						{isLoading ? "Loading..." : hasError ? "Error" : formatCpuUsage(metricsData.metrics.cpu)}
+					</Dd>
+					<Dt>Memory Usage</Dt>
+					<Dd className={hasError ? "text-destructive" : ""}>
+						{isLoading ? "Loading..." : hasError ? "Error" : formatMemoryUsage(metricsData.metrics.memory)}
+					</Dd>
+				</Dl>
+			</Flex>
+		</div>
+	);
+}