add telemetry agent and local backend collector for testing

docker/telemetry-collector/Dockerfile

@@ -0,0 +1,13 @@
+FROM otel/opentelemetry-collector-contrib:latest
+
+# Define an argument for the config file path
+ARG CONFIG_FILE=otel-collector-config.yaml
+
+# Copy the config file into the container
+COPY ${CONFIG_FILE} /otel-collector-config.yaml
+
+# Expose the necessary ports (health check, grpc, and http ports)
+EXPOSE 13134 4319 4316
+
+# Set the command to run the collector with the config file
+CMD ["--config", "/otel-collector-config.yaml"]

docker/telemetry-collector/otel-collector-config.yaml

@@ -0,0 +1,34 @@
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13134
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4316
+      http:
+        endpoint: 0.0.0.0:4319
+
+exporters:
+  logging:
+    verbosity: detailed
+
+processors:
+  batch:
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]

requirements.txt

@@ -1,6 +1,9 @@
 python-dotenv
 fastapi
 pexpect
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp
 pyopenssl>=23.3.0
 ray[default] >= 2.9.0
 rich

runhouse/constants.py

@@ -85,3 +85,14 @@
 DEFAULT_SURFACED_LOG_LENGTH = 20
 # Constants for schedulers
 INCREASED_INTERVAL = 1 * HOUR
+
+# Telemetry constants
+OTEL_VERSION = "0.108.0"
+
+TELEMETRY_AGENT_HTTP_PORT = 4318
+TELEMETRY_AGENT_GRPC_PORT = 4317
+TELEMETRY_AGENT_HEALTH_CHECK_PORT = 13133
+
+TELEMETRY_COLLECTOR_HOST = "telemetry.run.house"
+TELEMETRY_COLLECTOR_ENDPOINT = f"{TELEMETRY_COLLECTOR_HOST}:443"
+TELEMETRY_COLLECTOR_STATUS_URL = f"https://{TELEMETRY_COLLECTOR_HOST}/status"

runhouse/servers/telemetry/__init__.py

@@ -0,0 +1,5 @@
+from .telemetry_agent import (
+    TelemetryAgentConfig,
+    TelemetryAgentExporter,
+    TelemetryCollectorConfig,
+)