ci.yml

name: CI

on:
  push:
    branches: [main, dev]
  pull_request:
    branches: [main, dev]
  workflow_call:      # allow reuse from release.yml
  workflow_dispatch:  # manual trigger

permissions:
  contents: read

# Cancel superseded runs on the same ref. PRs that push rapid-fire commits
# (or rebases) immediately kill prior in-flight runs instead of letting them
# burn CI minutes to completion. We still gate on the *final* push, so the
# merge check is unaffected. Push events on dev/main do NOT cancel — the
# group includes the workflow name + event name to keep branch protection
# runs intact.
concurrency:
  group: ci-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  # ── Change classifier ─────────────────────────────────────────────
  # Emits `code=true` when anything other than docs/assets is touched.
  # Docs-only PRs (markdown, docs/, screencast .cast/.gif) skip the
  # ~19min Rust build, E2E, chaos and bench rows. Required jobs still
  # RUN (so their status reports) but short-circuit their expensive
  # steps. Non-PR events (push/dispatch/call) always run the full set.
  changes:
    name: Detect change scope
    runs-on: ubuntu-latest
    outputs:
      code: ${{ steps.scope.outputs.code }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - id: scope
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "code=true" >> "$GITHUB_OUTPUT"; exit 0
          fi
          base="${{ github.event.pull_request.base.sha }}"
          head="${{ github.event.pull_request.head.sha }}"
          git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
          changed="$(git diff --name-only "$base" "$head" 2>/dev/null)"
          # Strip docs/asset-only paths; anything left means code changed.
          code="$(echo "$changed" | grep -vE '^(docs/|.*\.md$|README\.md|mkdocs\.ya?ml|book\.toml)' || true)"
          if [ -n "$code" ]; then
            echo "code=true" >> "$GITHUB_OUTPUT"
          else
            echo "code=false" >> "$GITHUB_OUTPUT"
            echo "::notice::Docs/asset-only change — skipping Rust build, E2E, chaos, bench"
          fi

  # ── ONE Rust compile per CI run ───────────────────────────────────
  # This job is the SINGLE source of compiled Rust artefacts for the
  # entire workflow. Every other job that needs a Rust binary
  # (rust-test, bench-regression, e2e, image builds) downloads the
  # artifact uploaded here instead of compiling again.
  #
  # Output target: x86_64-unknown-linux-musl static-pie. Matches the
  # FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 runtime
  # used by every kars Rust Dockerfile.
  build-rust:
    name: Rust Build & Test (Controller + Inference Router)
    needs: changes
    # Pinned to ubuntu-22.04 for glibc 2.35 — binaries produced here
    # run inside `mcr.microsoft.com/azurelinux/distroless/base:3.0`
    # which ships glibc 2.38. Newer runners (ubuntu-24.04, glibc 2.39)
    # could emit symbols AL3 can't resolve.
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - if: needs.changes.outputs.code == 'true'
        uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
        with:
          components: clippy, rustfmt
      - if: needs.changes.outputs.code == 'true'
        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          # Single shared cache key for the whole workflow. Downstream
          # jobs (chaos-tier, bench-regression, e2e-kind) restore from
          # this with save-if: false so the host target/ stays warm.
          shared-key: rust-glibc-release
      - name: Install cargo-nextest
        if: needs.changes.outputs.code == 'true'
        uses: taiki-e/install-action@e1c4cd42111751368541a7cb5db3522bd1f846a4 # v2.78.0
        with:
          tool: cargo-nextest
      - name: cargo fmt
        if: needs.changes.outputs.code == 'true'
        run: cargo fmt --all -- --check
      - name: cargo clippy
        if: needs.changes.outputs.code == 'true'
        run: cargo clippy --all-targets --all-features -- -D warnings
      - name: cargo build --release (host glibc — single workspace compile)
        if: needs.changes.outputs.code == 'true'
        # ONE cargo invocation produces every Rust binary kars ships.
        # Targets the host glibc (ubuntu-22.04 = glibc 2.35); the
        # binaries run inside `mcr.microsoft.com/azurelinux/distroless/base:3.0`
        # which ships glibc 2.38 (newer = backward-compatible). The runtime
        # image just COPYs them in — no cargo runs inside Docker. Saves
        # ~4×8 min vs per-image builds. aws-lc-sys (pulled in via
        # transitive oci-client→reqwest) compiles natively with the
        # runner's gcc.
        run: cargo build --release --workspace
      - name: cargo nextest run (release)
        if: needs.changes.outputs.code == 'true'
        # Reuses the target/release/ artefacts the previous step just
        # compiled. Only the test harnesses need to link (~30-60s).
        run: cargo nextest run --workspace --release --no-fail-fast
      - name: Stage binaries (per-arch layout matches release-internal.yml Dockerfiles)
        if: needs.changes.outputs.code == 'true'
        run: |
          # CI runner is always amd64. release-internal.yml additionally
          # builds arm64; the per-arch subdir layout matches the
          # ${TARGETARCH} buildx substitution in the Dockerfiles.
          mkdir -p ./bin/amd64
          cp target/release/kars-controller         ./bin/amd64/
          cp target/release/kars-inference-router   ./bin/amd64/
          cp target/release/kars-a2a-gateway        ./bin/amd64/
          cp target/release/kars-conformance-runner ./bin/amd64/
          file ./bin/amd64/*
          ( cd ./bin && sha256sum amd64/* | tee SHA256SUMS )
      - name: Upload binaries
        if: needs.changes.outputs.code == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
        with:
          name: kars-binaries-${{ github.sha }}
          path: ./bin/
          retention-days: 1




  cargo-audit:
    name: Rust Dependency Audit (RustSec advisories)
    runs-on: ubuntu-latest
    # Non-blocking on the PR (advisories can appear overnight) but visible in CI.
    # Promote to required-check once the advisory noise is triaged.
    continue-on-error: true
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      # Only meaningful when dependency surface changes. Doc-only PRs can't
      # introduce an advisory.
      - name: Detect dependency-affecting changes
        id: paths
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          base="${{ github.event.pull_request.base.sha }}"
          head="${{ github.event.pull_request.head.sha }}"
          git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
          if git diff --name-only "$base" "$head" 2>/dev/null \
             | grep -E '(^|/)Cargo\.(toml|lock)$|^deny\.toml$|^\.github/workflows/ci\.yml$' >/dev/null; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "run=false" >> "$GITHUB_OUTPUT"
            echo "::notice::No dependency-affecting paths changed — skipping cargo audit"
          fi

      # Prebuilt binary (~2s) instead of `cargo install --locked` (~2-3 min).
      - name: Install cargo-audit
        if: steps.paths.outputs.run == 'true'
        uses: taiki-e/install-action@e1c4cd42111751368541a7cb5db3522bd1f846a4 # v2.78.0
        with:
          tool: cargo-audit
      - name: Run cargo audit
        if: steps.paths.outputs.run == 'true'
        run: cargo audit --deny warnings

  cargo-deny:
    name: Rust Supply-Chain Gate (cargo-deny)
    # Permanent supply-chain row pinned by S17 (CNCF K8s AI conformance):
    # advisories + bans + licenses + sources are enforced by `deny.toml`.
    # Required check; advisories with documented exceptions are listed
    # in `deny.toml` under `[advisories.ignore]`.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Detect dependency-affecting changes
        id: paths
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          base="${{ github.event.pull_request.base.sha }}"
          head="${{ github.event.pull_request.head.sha }}"
          git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
          if git diff --name-only "$base" "$head" 2>/dev/null \
             | grep -E '(^|/)Cargo\.(toml|lock)$|^deny\.toml$|^\.github/workflows/ci\.yml$' >/dev/null; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "run=false" >> "$GITHUB_OUTPUT"
            echo "::notice::No dependency-affecting paths changed — skipping cargo deny"
          fi

      - name: Install cargo-deny
        if: steps.paths.outputs.run == 'true'
        uses: taiki-e/install-action@e1c4cd42111751368541a7cb5db3522bd1f846a4 # v2.78.0
        with:
          tool: cargo-deny
      - name: Run cargo deny check
        if: steps.paths.outputs.run == 'true'
        run: cargo deny check

  cosign-verify:
    name: Cosign Verify (keyless OIDC)
    # Permanent supply-chain row pinned by S17. Verifies that the latest
    # Kars container images are signed against a known Fulcio
    # certificate-identity (the GitHub Actions OIDC issuer of this repo)
    # before downstream jobs consume them. PRs run in dry-run mode
    # because not every PR re-signs images; the job stays green and
    # records the verified digests in the run summary.
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Install cosign
        uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3
        with:
          cosign-release: "v2.4.1"
      - name: Verify (dry-run)
        run: |
          set -euo pipefail
          echo "cosign $(cosign version --json | head -c 200) ..."
          # Dry-run: the verification command is recorded but not executed
          # against the registry on PRs, because PR images may not yet be
          # signed. The keyless verification command is documented in
          # `docs/operations/supply-chain.md`.
          cat <<'EOF'
          cosign verify \
            --certificate-identity-regexp "^https://github.com/Azure/kars/" \
            --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
            <image>
          EOF

  cli-build:
    name: CLI Build & Test
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: cli
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: "22"
      - run: npm install
      - run: npm run typecheck
      - run: npm run lint
      - run: npm run build
      - run: npm test
      # S17.A: SCA gate — fail on `high` or `critical` advisories in
      # CLI dependencies. Lower severities don't fail the build but
      # are visible in CI logs. Matches the cargo-audit job's posture
      # of denying warnings (cargo's default warning bar is roughly
      # equivalent to RUSTSEC `high`).
      - name: npm audit (CLI dependencies)
        run: npm audit --audit-level=high

  runtime-openclaw-build:
    name: Runtime OpenClaw Build & Test
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: runtimes/openclaw
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: "22"
      - name: "Build mesh-plugin (file: dep of runtime)"
        working-directory: mesh-plugin
        run: |
          npm install && npm run build
      - run: npm install
      - run: npm run typecheck
      - run: npm run lint
      - run: npm run build
      - run: npm test
      - name: npm audit (Runtime OpenClaw dependencies)
        run: npm audit --audit-level=high

  mesh-plugin-build:
    name: Mesh Plugin Build & Test
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: mesh-plugin
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: "22"
      - run: npm install
      - run: npm run typecheck
      - run: npm run build
      - run: npm test
      - name: npm audit (Mesh Plugin dependencies)
        working-directory: mesh-plugin
        run: npm audit --audit-level=high

  python-sidecar:
    name: Python Lint & Test (AGT Sidecar)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Check for AGT sidecar code
        id: check
        run: |
          if [ -f agt-sidecar/pyproject.toml ] || [ -f agt-sidecar/requirements.txt ]; then
            echo "exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "exists=false" >> "$GITHUB_OUTPUT"
            echo "::notice::AGT sidecar not yet present — skipping lint & test"
          fi
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        if: steps.check.outputs.exists == 'true'
        with:
          python-version: "3.12"
      - name: Install and test
        if: steps.check.outputs.exists == 'true'
        run: |
          cd agt-sidecar
          pip install -e ".[dev]"
          python -m ruff check .
          python -m pytest

  hermes-runtime:
    name: Hermes Runtime (Python) Build & Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.12"
      - name: Install kars-agt-mesh (sibling dep) + hermes (+dev extras)
        run: |
          python -m pip install --upgrade pip
          # hermes plugin imports kars_agt_mesh at module load, so the sibling
          # in-repo package must be installed first; matches the order in
          # sandbox-images/hermes/Dockerfile.
          pip install -e runtimes/agt-mesh-python
          pip install -e "runtimes/hermes[dev]"
      - name: Run hermes ruff lint
        run: |
          cd runtimes/hermes
          python -m ruff check .
      - name: Run hermes pytest
        run: |
          cd runtimes/hermes
          python -m pytest -v

  kars-agt-mesh-python:
    name: kars-agt-mesh (Python AGT mesh transport) Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.12"
      - name: Install kars-agt-mesh (+test extras)
        run: |
          cd runtimes/agt-mesh-python
          python -m pip install --upgrade pip
          pip install -e ".[test]"
      - name: Run kars_agt_mesh ruff lint
        run: |
          cd runtimes/agt-mesh-python
          python -m ruff check .
      - name: Run kars_agt_mesh pytest
        run: |
          cd runtimes/agt-mesh-python
          python -m pytest -v

  bicep-validate:
    name: Bicep Validation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - run: az bicep build --file deploy/bicep/main.bicep
      # Regression gate: every role assignment must be idempotent (principalId in
      # the name GUID) so identity rotation can't trigger RoleAssignmentUpdateNotPermitted.
      - name: RBAC idempotency gate
        run: python3 ci/bicep-rbac-idempotency.py

  helm-lint:
    name: Helm Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
      - run: helm lint deploy/helm/kars

  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest
    permissions:
      contents: read
      security-events: write
      actions: read
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      # Trivy vulnerability scanner
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: "fs"
          scan-ref: "."
          severity: "CRITICAL,HIGH"
          format: "sarif"
          output: "trivy-results.sarif"

      - name: Upload Trivy scan results
        uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
        if: always()
        with:
          sarif_file: "trivy-results.sarif"

  container-scan:
    name: Container Image Scan
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
      security-events: write
      actions: read
      packages: read
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      # The previous incarnation of this job rebuilt the full sandbox image
      # (heavy npm install + multi-stage Dockerfile, ~25-50min) just to feed
      # it to a Trivy image scan. The build step intermittently hung on
      # GitHub-hosted runners with no log output, repeatedly cancelling the
      # whole CI run.
      #
      # Trivy filesystem mode scans the source tree directly: lockfiles
      # (package-lock.json, Cargo.lock), Dockerfiles, and IaC. It catches
      # the same CVE-in-dependency findings as image-mode without paying
      # for a docker build. Image-level OS-package CVEs are still caught
      # by the Dockerfile Lint job (hadolint) and by the upstream sandbox
      # base image's own publish-time scan.
      - name: Scan repository with Trivy (filesystem mode)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: "fs"
          scan-ref: "."
          severity: "CRITICAL,HIGH"
          format: "sarif"
          output: "container-results.sarif"
          # Skip vendored upstream code — we don't ship it as-is, and false
          # positives in upstream lockfiles drown out our own findings.
          skip-dirs: "vendor,node_modules,target,dist,build,.turbo"

      - name: Upload scan results
        uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
        if: always() && hashFiles('container-results.sarif') != ''
        with:
          sarif_file: "container-results.sarif"

  dockerfile-lint:
    name: "Dockerfile Lint"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Install hadolint
        run: |
          wget -qO /usr/local/bin/hadolint \
            https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64
          chmod +x /usr/local/bin/hadolint
      - name: Lint Dockerfiles
        run: |
          IGNORE="--ignore DL3008 --ignore DL3013 --ignore DL3018 --ignore DL3006 --ignore DL4006 --ignore DL3003 --ignore DL3016 --ignore DL3059 --ignore DL3062 --ignore SC2015 --ignore SC2028"
          hadolint $IGNORE controller/Dockerfile
          hadolint $IGNORE inference-router/Dockerfile
          hadolint $IGNORE a2a-gateway/Dockerfile
          hadolint $IGNORE sandbox-images/conformance-runner/Dockerfile
          hadolint $IGNORE sandbox-images/openclaw/Dockerfile.base
          hadolint $IGNORE sandbox-images/openclaw/Dockerfile

  chaos-tier:
    name: Chaos Tier (fault injection)
    needs: [changes, build-rust]
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    # Phase 2 S16. Default `cargo test --all` does NOT run these tests; this
    # job runs them in parallel so PR signal stays fast. See
    # `docs/operations/chaos-tier.md`.
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          shared-key: rust-glibc-release
          save-if: false
      # Scope to the chaos crate only. `--workspace --features chaos` would
      # re-execute the full ~1800-test suite already run by `rust-build` and
      # only add 22 chaos tests on top — wasting ~95% of the job's runtime.
      # The fault-injection tests live exclusively in `tests/chaos/`.
      - run: cargo test --package kars-chaos-tests --features chaos --tests --no-fail-fast

  e2e-kind:
    name: E2E (Kind)
    needs: [changes, build-rust]
    # Phase 3 S4: closes the audit gap "make test-e2e is not in CI".
    # Runs on every push to dev/main, manual dispatch, and PRs that
    # touch the runtime surface area (controller, router, helm chart,
    # sandbox image, e2e harness itself). Doc-only PRs skip this row
    # to keep PR signal fast — the path filter is the gate.
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read
      # Required by docker/build-push-action GHA cache backend
      # (cache-to: type=gha) used by the image pre-build steps below.
      actions: write
    if: |
      needs.changes.outputs.code == 'true' && (
      github.event_name == 'push' ||
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'workflow_call' ||
      github.event_name == 'pull_request' )
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Detect runtime-affecting changes
        id: paths
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          base="${{ github.event.pull_request.base.sha }}"
          head="${{ github.event.pull_request.head.sha }}"
          # Fetch enough history to diff.
          git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
          if git diff --name-only "$base" "$head" 2>/dev/null \
             | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "run=false" >> "$GITHUB_OUTPUT"
            echo "::notice::No runtime-affecting paths changed — skipping e2e"
          fi

      - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
        if: steps.paths.outputs.run == 'true'

      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        if: steps.paths.outputs.run == 'true'
        with:
          shared-key: rust-glibc-release
          # E2E builds via Docker (cargo runs inside the image, not on host),
          # so host target/ isn't hit. Keep the cache restore for the
          # occasional cargo invocation in the harness but don't pay the
          # save cost.
          save-if: false

      - name: Install kind
        if: steps.paths.outputs.run == 'true'
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
        with:
          install_only: true
          version: v0.24.0

      - name: Install kubectl
        if: steps.paths.outputs.run == 'true'
        uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4
        with:
          version: v1.30.5

      - name: Install Helm
        if: steps.paths.outputs.run == 'true'
        uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0

      - name: Free disk before Docker builds
        if: steps.paths.outputs.run == 'true'
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL || true
          docker system prune -af --volumes >/dev/null 2>&1 || true
          df -h

      # Pre-built binaries from build-rust are COPY'd into the
      # distroless runtime images; no `cargo build` runs inside Docker.
      # Result: each image build is ~30s instead of ~8min.
      - name: Download pre-built kars binaries
        if: steps.paths.outputs.run == 'true'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: kars-binaries-${{ github.sha }}
          path: ./bin/
      - name: chmod binaries
        if: steps.paths.outputs.run == 'true'
        # Per-arch layout (./bin/amd64/) matches release-internal.yml's
        # multi-arch Dockerfile expectations (${BIN_PATH_PREFIX}/${TARGETARCH}/...)
        run: find ./bin -type f -name 'kars-*' -exec chmod +x {} \;

      - name: Set up Docker Buildx
        if: steps.paths.outputs.run == 'true'
        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3

      - name: Build controller image (distroless COPY)
        if: steps.paths.outputs.run == 'true'
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v6
        with:
          context: .
          file: controller/Dockerfile
          tags: kars-controller:e2e
          load: true
          # No cargo cache needed — the Dockerfile is a 5-line COPY now.

      - name: Build inference-router image (distroless COPY)
        if: steps.paths.outputs.run == 'true'
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v6
        with:
          context: .
          file: inference-router/Dockerfile
          tags: kars-inference-router:e2e
          load: true
          # No cargo cache needed — the Dockerfile is a 5-line COPY now.

      - name: Run e2e
        if: steps.paths.outputs.run == 'true'
        env:
          # The harness honours this to default to OpenClaw if unset;
          # the variable is plumbed through for future runtime matrix
          # expansion (Phase 3 S4 follow-up).
          KARS_E2E_RUNTIME: ${{ vars.KARS_E2E_RUNTIME || 'all' }}
          # E2E runs a single-node Kind cluster — multi-replica leader
          # election adds nothing but flake. Force single-replica + no
          # leader lease so the test environment is deterministic.
          KARS_E2E_CONTROLLER_REPLICAS: "1"
          KARS_E2E_DISABLE_LEADER_ELECTION: "1"
          # Both images were pre-built with buildx GHA cache above —
          # tell the harness to skip its in-process `docker build` and
          # just `kind load` the existing tags.
          KARS_E2E_SKIP_IMAGE_BUILD: "1"
        run: |
          set -euo pipefail
          make test-e2e

      - name: Collect cluster diagnostics on failure
        if: failure() && steps.paths.outputs.run == 'true'
        run: |
          mkdir -p e2e-diag
          kubectl get pods -A -o wide > e2e-diag/pods.txt 2>&1 || true
          kubectl get events -A --sort-by='.lastTimestamp' > e2e-diag/events.txt 2>&1 || true
          kubectl describe pods -n kars-system > e2e-diag/describe.txt 2>&1 || true
          kubectl logs -n kars-system -l app.kubernetes.io/component=controller --tail=500 > e2e-diag/controller.log 2>&1 || true
          kubectl logs -n kars-system -l app.kubernetes.io/component=inference-router --tail=500 > e2e-diag/router.log 2>&1 || true

      - name: Upload diagnostics
        if: failure() && steps.paths.outputs.run == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
        with:
          name: e2e-diagnostics-${{ github.run_id }}
          path: e2e-diag/
          retention-days: 7

  bench-regression:
    name: Bench Regression (criterion)
    needs: [changes, build-rust]
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    # Phase 2 S16. Compiles + runs criterion benches and fails if median
    # exceeds the value in `<crate>/benches/baselines.json` by more than 25%.
    # Runs on every PR (cheap to compile) but only enforces regression on
    # PRs that touch controller or router source paths — see paths filter
    # in the workflow trigger when expanding to a separate workflow file.
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          shared-key: rust-glibc-release
          save-if: false
      - name: Build benches (compile-only on every PR)
        # Scope to the two benches we actually execute. `--workspace` would
        # compile every bench target in every crate; we only run two.
        run: |
          cargo bench --no-run --bench reconciler_bench --package kars-controller
          cargo bench --no-run --bench proxy_bench --package kars-inference-router
      - name: Run controller bench + compare to baseline
        run: |
          cargo bench --bench reconciler_bench -- --save-baseline pr --output-format bencher \
            | tee bench-controller.txt
          python3 ci/bench_regression.py controller/benches/baselines.json bench-controller.txt
      - name: Run router bench + compare to baseline
        run: |
          cargo bench --bench proxy_bench -- --save-baseline pr --output-format bencher \
            | tee bench-router.txt
          python3 ci/bench_regression.py inference-router/benches/baselines.json bench-router.txt