feat(actors): expose container system metrics

Author: NathanFlurry

docker/dev-full/docker-compose.yml

@@ -233,6 +233,8 @@ services:
       # We only reserve 100 ports instead of the default 22,000. See
       # rivet-guard for explanation.
       - "7600-7699:7600-7699"
+      # cAdvisor metrics endpoint
+      - "7780:7780"
     networks:
       - rivet-network
 

docker/dev-full/otel-collector/config.yaml

@@ -5,6 +5,14 @@ receivers:
         endpoint: 0.0.0.0:4317
       http:
         endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'cadvisor'
+          static_configs:
+            - targets: ['rivet-client:7780']
+          metrics_path: /metrics
+          scrape_interval: 30s
 
 processors:
   batch:
@@ -52,7 +60,7 @@ service:
       processors: [batch]
       exporters: [clickhouse]
     metrics:
-      receivers: [otlp]
+      receivers: [otlp, prometheus]
       processors: [batch]
       exporters: [clickhouse]
 

docker/universal/Dockerfile

@@ -120,8 +120,8 @@ RUN apt-get update -y && \
     apt-get install -y ca-certificates openssl curl tini curl && \
     curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.1.60/libfdb_c.x86_64.so"
 
-# MARK: Runner (Full)
-FROM --platform=linux/amd64 base-runner AS client-full
+# MARK: Runner (Slim)
+FROM --platform=linux/amd64 base-runner AS client-slim
 ARG CNI_PLUGINS_VERSION=1.3.0
 RUN apt-get install -y skopeo iproute2 runc dnsutils && \
 	echo "Downloading lz4" && \
@@ -142,6 +142,23 @@ COPY ./docker/dev-full/rivet-client/rivet-actor.conflist /opt/cni/config/rivet-a
 COPY --from=builder /app/dist/rivet-client /app/dist/rivet-container-runner /usr/local/bin/
 ENTRYPOINT ["/usr/bin/tini", "--", "entrypoint.sh"]
 
+# MARK: Runner (Full)
+FROM client-slim AS client-full
+ARG CADVISOR_VERSION=v0.52.0
+RUN apt-get update -y && \
+    apt-get install -y wget && \
+    wget -O /usr/local/bin/cadvisor "https://github.com/google/cadvisor/releases/download/${CADVISOR_VERSION}/cadvisor-${CADVISOR_VERSION}-linux-amd64" && \
+    chmod +x /usr/local/bin/cadvisor && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY docker/universal/client-full-entrypoint.sh /usr/local/bin/client-full-entrypoint.sh
+RUN chmod +x /usr/local/bin/client-full-entrypoint.sh
+
+EXPOSE 7780
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/client-full-entrypoint.sh"]
+
 # MARK: Monlith
 FROM --platform=linux/amd64 debian:12.9-slim AS monolith
 ENV DEBIAN_FRONTEND=noninteractive

docker/universal/client-full-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# Start cadvisor in the background
+cadvisor \
+  --port=7780 \
+  --listen_ip=0.0.0.0 \
+  --prometheus_endpoint="/metrics" \
+  --enable_metrics=cpu,cpuLoad,memory,network,disk,diskIO,oom_event,process,tcp,udp \
+  --docker_only=false \
+  --disable_root_cgroup_stats=false &
+
+# TODO:
+# --raw_cgroup_prefix_whitelist="" \
+
+# Start rivet-client with all passed arguments
+exec rivet-client "$@"

packages/common/api-helper/build/src/macro_util.rs

@@ -237,7 +237,7 @@ pub fn __deserialize_query<T: DeserializeOwned + Send>(route: &Url) -> GlobalRes
 }
 
 #[doc(hidden)]
-#[tracing::instrument(skip_all, name="setup_ctx")]
+#[tracing::instrument(skip_all, name = "setup_ctx")]
 pub async fn __with_ctx<
 	A: auth::ApiAuth + Send,
 	DB: chirp_workflow::db::Database + Sync + 'static,

packages/common/clickhouse-inserter/src/error.rs

@@ -2,15 +2,15 @@ use thiserror::Error;
 
 #[derive(Debug, Error)]
 pub enum Error {
-    #[error("failed to send event to ClickHouse inserter")]
-    ChannelSendError,
+	#[error("failed to send event to ClickHouse inserter")]
+	ChannelSendError,
 
-    #[error("serialization error: {0}")]
-    SerializationError(#[source] serde_json::Error),
+	#[error("serialization error: {0}")]
+	SerializationError(#[source] serde_json::Error),
 
-    #[error("failed to build reqwest client: {0}")]
-    ReqwestBuildError(#[source] reqwest::Error),
+	#[error("failed to build reqwest client: {0}")]
+	ReqwestBuildError(#[source] reqwest::Error),
 
-    #[error("failed to spawn background task")]
-    TaskSpawnError,
-}
+	#[error("failed to spawn background task")]
+	TaskSpawnError,
+}

packages/common/config/src/config/server/rivet/mod.rs

@@ -177,7 +177,7 @@ impl Rivet {
 	}
 }
 
-impl Rivet {	
+impl Rivet {
 	pub fn default_cluster_id(&self) -> GlobalResult<Uuid> {
 		if let Some(default_cluster_id) = self.default_cluster_id {
 			ensure!(
@@ -190,7 +190,9 @@ impl Rivet {
 				// Return default development clusters
 				AccessKind::Development => Ok(default_dev_cluster::CLUSTER_ID),
 				// No cluster configured
-				AccessKind::Public | AccessKind::Private => bail!("`default_cluster_id` not configured"),
+				AccessKind::Public | AccessKind::Private => {
+					bail!("`default_cluster_id` not configured")
+				}
 			}
 		}
 	}

packages/common/formatted-error/errors/actor/metrics/invalid-metrics.md

@@ -0,0 +1,9 @@
+---
+name = "ACTOR_METRICS_INVALID_METRICS"
+description = "Invalid metrics format."
+http_status = 400
+---
+
+# Invalid Metrics
+
+The provided list of metrics is not in a valid JSON format. Please provide a valid JSON array of metric names.

packages/common/formatted-error/errors/actor/metrics/no-metrics.md

@@ -0,0 +1,9 @@
+---
+name = "ACTOR_METRICS_NO_METRICS"
+description = "No metrics specified."
+http_status = 400
+---
+
+# No Metrics
+
+No metrics were specified in the request. Please provide at least one metric name to query.

packages/common/formatted-error/errors/actor/metrics/unsupported-metrics.md

@@ -0,0 +1,9 @@
+---
+name = "ACTOR_METRICS_UNSUPPORTED_METRICS"
+description = "Unsupported metrics requested."
+http_status = 400
+---
+
+# Unsupported Metrics
+
+The requested metrics are not supported. Supported metrics include: cpu, memory, memory_limit, network_rx_bytes, network_tx_bytes.