Claude/implement e2e plan yt1o c

Dockerfile

@@ -20,7 +20,8 @@ RUN apk add --no-cache \
     tzdata \
     bash \
     coreutils \
-    curl
+    curl \
+    jq
 
 # Create non-root user (UID 10001 to avoid conflicts with system UIDs)
 RUN adduser -D -u 10001 agent

deploy/e2e/.env.template

@@ -0,0 +1,2 @@
+# Copy to .env and fill in. .env is gitignored.
+ANTHROPIC_API_KEY=

deploy/e2e/.gitignore

@@ -0,0 +1,3 @@
+.env
+agent-investigations/
+*.log

deploy/e2e/agent-config/alert-sources.yaml

@@ -0,0 +1,6 @@
+addr: ":8080"
+
+sources:
+  - type: prometheus
+    name: alertmanager
+    webhook_path: /alerts/prometheus

deploy/e2e/agent-config/skills/e2e-target/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: e2e-target
+description: E2E test target VM access — connection details and Prometheus query helpers for the load-test target.
+allowed-tools: bash read_file list_files fetch
+---
+
+# E2E Target VM
+
+You are investigating an alert from an end-to-end test environment. The
+following infrastructure is reachable on the `e2e-net` Docker network:
+
+- **Target VM**: hostname `target` (container `e2e-target`)
+  - node_exporter metrics: `http://target:9100/metrics`
+  - Demo HTTP service: `http://target:8080/`
+- **Prometheus**: `http://prometheus:9090`
+  - Instant query: `http://prometheus:9090/api/v1/query?query=...`
+  - Range query: `http://prometheus:9090/api/v1/query_range?query=...`
+- **Alertmanager**: `http://alertmanager:9093`
+  - Active alerts: `http://alertmanager:9093/api/v2/alerts`
+
+## Investigation Cheatsheet
+
+```bash
+# Current CPU utilization (%)
+curl -s 'http://prometheus:9090/api/v1/query?query=100-avg(rate(node_cpu_seconds_total{mode="idle"}[1m]))*100'
+
+# Current memory utilization (%)
+curl -s 'http://prometheus:9090/api/v1/query?query=(1-node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes)*100'
+
+# Current load average (1m)
+curl -s 'http://prometheus:9090/api/v1/query?query=node_load1'
+
+# Disk read throughput
+curl -s 'http://prometheus:9090/api/v1/query?query=rate(node_disk_read_bytes_total[1m])'
+
+# Confirm node_exporter is up
+curl -s http://target:9100/metrics | head -20
+```
+
+When you have enough evidence, produce an RCA finding with a confidence
+score. The expected root cause for this test is synthetic CPU/memory
+stress generated by a `stress-ng` workload in the `e2e-load-gen` container.

deploy/e2e/alertmanager/alertmanager.yml

@@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: code-agent
+  group_wait: 5s
+  group_interval: 10s
+  repeat_interval: 1m
+
+receivers:
+  - name: code-agent
+    webhook_configs:
+      - url: "http://code-agent:8080/alerts/prometheus"
+        send_resolved: true

deploy/e2e/docker-compose.yml

@@ -0,0 +1,89 @@
+services:
+  target:
+    build:
+      context: ./target
+      dockerfile: Dockerfile
+    container_name: e2e-target
+    ports:
+      - "9100:9100"   # node_exporter
+      - "8080:8080"   # naive demo HTTP service
+    networks:
+      - e2e-net
+    labels:
+      env: e2e-test
+
+  prometheus:
+    image: prom/prometheus:v3.3.1
+    container_name: e2e-prometheus
+    volumes:
+      - ./prometheus:/etc/prometheus:ro
+    ports:
+      - "9090:9090"
+    networks:
+      - e2e-net
+    depends_on:
+      - target
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1
+    container_name: e2e-alertmanager
+    volumes:
+      - ./alertmanager:/etc/alertmanager:ro
+    ports:
+      - "9093:9093"
+    networks:
+      - e2e-net
+    depends_on:
+      - prometheus
+
+  code-agent:
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+    container_name: e2e-code-agent
+    env_file:
+      - .env
+    environment:
+      # Anthropic SDK reads ANTHROPIC_API_KEY from env_file.
+      # AGENT_* prefix maps to internal config (see CLAUDE.md).
+      - AGENT_LOG_LEVEL=debug
+      - AGENT_LOG_FORMAT=json
+      - AGENT_SAFETY_COMMAND_VALIDATION_MODE=blacklist
+      - AGENT_MODEL=claude-haiku-4-5-20251001
+    # Pass --config explicitly because the image ships
+    # config/alert-sources.example.yaml, not alert-sources.yaml.
+    command:
+      - serve
+      - --config
+      - /app/config/alert-sources.yaml
+      - --auto-approve-safe
+    volumes:
+      # Alert sources YAML mounted at the path serve --config points to.
+      - ./agent-config/alert-sources.yaml:/app/config/alert-sources.yaml:ro
+      # Skills are loaded from ./skills, ./.claude/skills, or ~/.claude/skills
+      # (LocalSkillManager hardcoded list). Mount under .claude/skills so we
+      # don't shadow the built-in /app/skills baked into the image.
+      - ./agent-config/skills:/app/.claude/skills:ro
+      # Investigation JSON lands at /app/.agent/investigations/.
+      - ./agent-investigations:/app/.agent
+    ports:
+      - "8090:8080"   # public webhook (avoid clash with target :8080)
+      - "8091:8081"   # internal /health, /ready
+    networks:
+      - e2e-net
+    depends_on:
+      - alertmanager
+
+  load-generator:
+    build:
+      context: ./load-gen
+      dockerfile: Dockerfile
+    container_name: e2e-load-gen
+    networks:
+      - e2e-net
+    depends_on:
+      - target
+
+networks:
+  e2e-net:
+    driver: bridge

deploy/e2e/load-gen/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.21
+
+# stress-ng lives in the community repository on Alpine.
+RUN apk add --no-cache bash curl coreutils stress-ng
+
+COPY generate-load.sh /app/generate-load.sh
+RUN chmod +x /app/generate-load.sh
+
+CMD ["/app/generate-load.sh"]

deploy/e2e/load-gen/generate-load.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "[load-gen] Phase 1: baseline idle (30s)..."
+sleep 30
+
+echo "[load-gen] Phase 2a: CPU stress (120s, all cores)..."
+stress-ng --cpu 0 --timeout 120s --metrics-brief &
+
+echo "[load-gen] Phase 2b: memory pressure (120s, 4x1G, --vm-keep to actually hold pages)..."
+stress-ng --vm 4 --vm-bytes 1G --vm-keep --timeout 120s &
+
+echo "[load-gen] Phase 2c: HTTP flood against target:8080 (500 reqs, 20 parallel)..."
+seq 1 500 | xargs -n 1 -P 20 -I{} curl -s -m 2 -o /dev/null http://target:8080/ || true
+
+wait
+echo "[load-gen] Stress phase complete. Sleeping forever for log inspection."
+sleep infinity

deploy/e2e/prometheus/alert_rules.yml

@@ -0,0 +1,32 @@
+groups:
+  - name: e2e-load-alerts
+    rules:
+      - alert: HighCPUUtilization
+        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[30s])) * 100) > 80
+        for: 15s
+        labels:
+          severity: critical
+          team: sre
+        annotations:
+          summary: "High CPU on {{ $labels.instance }}"
+          description: "CPU usage is {{ $value }}% on {{ $labels.instance }} — above 80% threshold"
+
+      - alert: HighMemoryUtilization
+        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
+        for: 15s
+        labels:
+          severity: critical
+          team: sre
+        annotations:
+          summary: "High memory on {{ $labels.instance }}"
+          description: "Memory usage is {{ $value }}% on {{ $labels.instance }}"
+
+      - alert: HighLoadAverage
+        expr: node_load1 > 2
+        for: 15s
+        labels:
+          severity: critical
+          team: sre
+        annotations:
+          summary: "High load average on {{ $labels.instance }}"
+          description: "1-minute load average is {{ $value }} on {{ $labels.instance }}"

deploy/e2e/prometheus/prometheus.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: target
+    static_configs:
+      - targets: ["target:9100"]
+        labels:
+          env: e2e-test
+          host: e2e-target
+
+rule_files:
+  - /etc/prometheus/alert_rules.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]

deploy/e2e/run-test.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+set -euo pipefail
+
+E2E_DIR="$(cd "$(dirname "$0")" && pwd)"
+cd "$E2E_DIR"
+
+# --- Pre-flight ---
+if [ -z "${ANTHROPIC_API_KEY:-}" ] && [ -f .env ]; then
+  # shellcheck disable=SC1091
+  set -a; source .env; set +a
+fi
+
+if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
+  echo "ERROR: ANTHROPIC_API_KEY must be set (env var or .env file)" >&2
+  exit 1
+fi
+
+if [ ! -f docker-compose.yml ]; then
+  echo "ERROR: docker-compose.yml missing. Run ./setup.sh first." >&2
+  exit 1
+fi
+
+mkdir -p agent-investigations
+# Ensure the non-root container user (UID 10001) can write here.
+chmod 0777 agent-investigations || true
+
+echo "=== E2E Load-Investigation Test ==="
+
+echo "[1/5] Cleaning up previous run..."
+podman compose down -v --remove-orphans 2>/dev/null || true
+rm -rf agent-investigations/*
+
+echo "[2/5] Building images..."
+podman compose build
+
+echo "[3/5] Starting stack..."
+podman compose up -d
+
+echo "[4/5] Waiting for code-agent /ready (max 120s)..."
+for i in $(seq 1 60); do
+  if curl -sf http://localhost:8091/ready >/dev/null 2>&1; then
+    echo "  agent ready after ${i} attempts"
+    break
+  fi
+  sleep 2
+  if [ "$i" -eq 60 ]; then
+    echo "ERROR: agent never became ready" >&2
+    podman compose logs code-agent | tail -50
+    exit 1
+  fi
+done
+
+echo "[5/5] Watching for investigation output (timeout 300s)..."
+echo "  load-gen begins stress at ~30s; alerts fire at ~45s."
+
+ELAPSED=0
+TIMEOUT=300
+while [ "$ELAPSED" -lt "$TIMEOUT" ]; do
+  # Investigation files land at agent-investigations/investigations/<id>.json.
+  # Success: at least one investigation reached a terminal state (completed or
+  # escalated) AND produced findings or escalated. Other investigations in the
+  # batch may still be in flight or have given up at the action budget — the
+  # pipeline works as long as one alert produced a useful RCA.
+  if python3 - <<'PY' 2>/dev/null
+import glob, json, sys
+files = glob.glob('agent-investigations/**/*.json', recursive=True)
+if not files:
+    sys.exit(1)
+for path in files:
+    try:
+        with open(path) as fh:
+            data = json.load(fh)
+    except (OSError, json.JSONDecodeError):
+        continue
+    if data.get('status') not in ('completed', 'escalated'):
+        continue
+    if data.get('findings') or data.get('escalated'):
+        sys.exit(0)
+sys.exit(1)
+PY
+  then
+    echo
+    echo "=== Investigation output ==="
+    while IFS= read -r -d '' f; do
+      echo "--- $f ---"
+      python3 -m json.tool < "$f" 2>/dev/null || cat "$f"
+      echo
+    done < <(find agent-investigations -type f -name '*.json' -print0)
+    echo "=== Test PASSED ==="
+    podman compose down -v
+    exit 0
+  fi
+  sleep 5
+  ELAPSED=$((ELAPSED + 5))
+  printf '  waited %ss / %ss\n' "$ELAPSED" "$TIMEOUT"
+done
+
+echo
+echo "=== TIMEOUT after ${TIMEOUT}s — no investigation produced ===" >&2
+echo
+echo "--- Prometheus alerts ---"
+curl -sf http://localhost:9090/api/v1/alerts 2>/dev/null | python3 -m json.tool 2>/dev/null \
+  || echo "(prometheus unreachable)"
+echo "--- Alertmanager alerts ---"
+curl -sf http://localhost:9093/api/v2/alerts 2>/dev/null | python3 -m json.tool 2>/dev/null \
+  || echo "(alertmanager unreachable)"
+echo "--- Investigation files (status + findings count) ---"
+if compgen -G 'agent-investigations/investigations/*.json' >/dev/null; then
+  for f in agent-investigations/investigations/*.json; do
+    jq -c --arg f "$f" '{file:$f, status, findings_len:(.findings|length), escalated, actions_taken}' "$f" 2>/dev/null \
+      || echo "(failed to parse $f)"
+  done
+else
+  echo "(no investigation files written)"
+fi
+echo "--- code-agent logs (tail 80) ---"
+podman compose logs --tail=80 code-agent
+echo
+echo "Containers left running for debugging. ./teardown.sh to clean up."
+exit 1