diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..372c21d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: ci + +on: + push: + branches: [dev] + pull_request: + branches: [dev] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: "1.25.x" + GOLANGCI_VERSION: "v2.12.2" + +jobs: + build-test-lint: + name: build · vet · gofmt · lint · test · e2e + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + cache-dependency-path: go.sum + + - name: Download and verify modules + run: | + go mod download + go mod verify + + - name: gofmt check + run: | + diff=$(gofmt -l .) + if [ -n "$diff" ]; then + echo "::error::gofmt needed on:" + echo "$diff" + exit 1 + fi + + - name: go vet + run: go vet ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@d583c34f0599d37dbac4a198b9c83201be380893 # v9.3.0 + with: + version: ${{ env.GOLANGCI_VERSION }} + args: ./... + + - name: Format and imports check + run: golangci-lint fmt --diff ./... + + - name: Build + run: go build -trimpath ./... + + - name: Unit tests with race detector + run: go test -race -count=1 -coverprofile=coverage.out ./... + + - name: Binary integration smoke test + run: go test -race -count=1 -tags=e2e ./tests/e2e diff --git a/.gitignore b/.gitignore index cbe3512..6e5ea02 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ bin/ vendor/ *.test *.out +coverage.* # Node / UI node_modules/ dist/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..1bf0774 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,43 @@ +version: "2" + +run: + timeout: 5m + tests: true + +linters: + default: none + enable: + - bodyclose + - errcheck + - gosec + - govet + - ineffassign + - misspell + - revive + - staticcheck + - unconvert + - unused + settings: + revive: + rules: + - name: exported + - name: package-comments + - name: error-strings + - name: context-as-argument + - name: unreachable-code + misspell: + locale: US + exclusions: + generated: lax + rules: + - path: _test\.go + linters: [errcheck, gosec] + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/ArdurAI/sith diff --git a/COMPETITIVE.md b/COMPETITIVE.md new file mode 100644 index 0000000..692a61b --- /dev/null +++ b/COMPETITIVE.md @@ -0,0 +1,124 @@ +# Sith — Competitive Landscape + +**Status:** planning · **Date:** 2026-07-08 · **All facts web-verified July 2026** (sources at end) + +This is an honest map, including where the space is **crowded**. The conclusion is *not* +"nobody does this" — several strong efforts converge on parts of it. The conclusion is that +**vendor-neutral, OSS, governed *action* federation as an adoptable primitive** — with the +AI as a client of the governance rather than the product — is a narrow but real gap, and +that Sith is defensible only if it stays narrow and gets the governance right. + +--- + +## 1. The substrate we adopt (not compete with) + +| Project | What it is | Verified status / version | Role for Sith | +|---|---|---|---| +| **Open Cluster Management (OCM)** | CNCF framework for multi-cluster management | **CNCF Sandbox** (accepted 2021-11-09); core `ocm` **v1.3.1** (2026-05-19) | The substrate — Sith builds on it | +| **OCM `cluster-proxy`** | Reverse tunnels managed→hub; hub reaches cluster-local services across isolated VPCs; automates konnectivity | **v0.10.0** (2026-02-02); "Service Proxy" added v0.9.0 | Adopted transport | +| **OCM `managed-serviceaccount`** | Syncs SAs to spokes, projects scoped tokens back to hub | **v0.10.0** (2026-02-02) | Adopted scoped identity | +| **Kubernetes SIG `apiserver-network-proxy` (Konnectivity)** | The underlying reverse-tunnel technique | Kubernetes SIG project | The pattern cluster-proxy automates | +| **Rancher `remotedialer`** | Outbound reverse tunnel (K3s/Rancher scale) | **v0.6.1** | Proves the pattern is commodity | + +**Takeaway:** the "outbound-only agent + hub reaches cluster-local services" mechanism is +**commodity, hardened, CNCF/vendor-maintained plumbing** — building it bespoke would be +re-implementing security-sensitive infrastructure worse than three existing efforts. So it +is **out of scope** ([ADR-0001](docs/adr/0001-adopt-ocm-vs-bespoke-tunnel.md)). + +## 2. Adjacent but different (clear category lines) + +| Project | Category | Verified | Why it is *not* Sith | +|---|---|---|---| +| **Karmada** | Multi-cluster **scheduling / workload placement** | CNCF; **v1.18.1** (2026-06-30) | Places workloads; Sith governs *operations*, not placement | +| **Clusterpedia** | Multi-cluster **resource inventory / search** | CNCF Sandbox; **v0.9.1** (2026-04-16) | Read/search only; no governed action federation | +| **Headlamp** | Single-/multi-cluster **UI** | Kubernetes-sigs; **v0.43.0** (2026-06-16) | A console, not a governed action control plane | +| **Devtron** | OSS K8s **platform** (CI/CD + GitOps + obs + security) | **v2.1.1** (2026-03-24) | Batteries-included platform, not a narrow federation-governance layer | +| **argocd-agent** | Agent-based **Argo CD** multi-cluster | argoproj-labs; **v0.9.0** (2026-06-04) | Federates Argo CD's *own* surface; Sith is tool-agnostic + governed | +| **kagent** | CNCF K8s-native **agent framework** | **v0.10.0-beta4** (2026-07-06) | A framework to *build* agents; a potential **MCP client** of Sith, not a competitor | + +## 3. The genuinely competitive zone (be honest — it is crowded) + +The product space *above* the plumbing is occupied and converging on "federate existing +tools + govern + put AI on top": + +- **Komodor** — Autonomous AI SRE. Shipped an **extensible multi-agent architecture** + (2026-03-18) letting orgs bring their own tools/services/agents via **MCP or OpenAPI**, + orchestrated alongside **50+ specialized agents**, with unified fleet visibility. This is + the closest thing to "federate existing cluster-local tools + AI", **shipping and + funded**. +- **SUSE Rancher Prime** — At **KubeCon EU 2026**, positioned as an **agentic AI + ecosystem**: the "Liz" assistant became a **crew** of agents (Security/Observability/ + Platform/SLE/App), with **external MCP server** plug-in so third-party tools become + "crew members", plus K3k virtual-cluster GPU multi-tenancy. Hyperscaler-scale, vendor + platform. +- **Devtron** — OSS, multi-cluster, AI-native platform at scale. + +**Honest read:** the *visibility + AI-assistant* layer is well-served and competitive. If +Sith tried to be "an AI SRE that sees your fleet", it would be entering Komodor's and +Rancher's strongest 2026 investment with none of their funding, data, or vendor weight. + +## 4. Where the wedge is defensible + +Sith does **not** win on visibility or on "most agents". It is defensible only on a narrow +axis the incumbents under-serve as an *adoptable, neutral primitive*: + +1. **Governed *action* federation, not just visibility.** The incumbents excel at *see* and + at *investigate*; safe, policy-gated, **typed-intent fan-out to N clusters** — with wave + gates, multi-approver prod, partial-failure/rollback, and **federation-specific + abstention** — is under-served as a clean primitive rather than one vendor's feature. +2. **AI as a *client of the governance*, not the product.** Komodor and Rancher put agents + *on top*. Sith inverts it: it is a **governed MCP server** where the org's own agent and + *any* external agent (Claude Code, Codex, kagent) inherit the **same** PDP, closed verb + vocabulary, scoped identity, and audit. "A governed MCP gateway to your whole fleet" is a + platform position, not a chatbot. +3. **Vendor-neutral, OSS, and deliberately narrow.** Not tied to one vendor's stack (SUSE) + or one SaaS (Komodor). Built on a CNCF substrate (OCM), tool-agnostic, and scoped so + tightly (no portal, no scheduler, no telemetry store, **no shell ever**) that it can be + *correct* where a broad platform can only be *featureful*. +4. **Correctness of isolation + action safety as the product.** The hard, valuable, + defensible thing is getting multi-tenant isolation, signed intents, per-spoke local + enforcement, and least-privilege identity **right** — precisely where broad platforms + carry more surface and more risk. + +## 5. Risks to the thesis (stated plainly) + +- **The gap is narrowing.** As OCM's addon ecosystem, argocd-agent, and vendor MCP surfaces + mature, the space compresses. Milestone-0 + Phase-1 must validate demand fast. +- **Governance-of-action is contested at the edges** (AI gateways, vendor MCP governance). + Sith's differentiation is the *fan-out policy* + *neutral OSS* + *action-federation* + intersection — thinner than a whole product, which is exactly why it must stay narrow. +- **Execution risk is the isolation problem itself.** The single hardest part (correct + multi-tenant isolation + safe action) is the part predecessors got wrong. This plan makes + those day-one, structural requirements rather than later hardening. + +## 6. One-line positioning + +> **Komodor/Rancher put AI *on top* of the fleet. Sith puts *governance* under the fleet — +> and lets any AI be a governed client of it.** + +--- + +## Sources (verified July 2026) + +- OCM — CNCF status: · project: + · core repo: +- OCM `cluster-proxy` (v0.10.0, reverse tunnels / cross-VPC / konnectivity): + · Service Proxy: + +- OCM `managed-serviceaccount` (v0.10.0): + · + +- Konnectivity / `apiserver-network-proxy`: +- Rancher `remotedialer`: +- Karmada (v1.18.1): · +- Clusterpedia (v0.9.1): +- Headlamp (v0.43.0): +- Devtron (v2.1.1): +- argocd-agent (v0.9.0): +- kagent (v0.10.0-beta4): · +- Komodor extensible multi-agent + MCP/OpenAPI (2026-03-18): + +- SUSE Rancher Prime agentic AI + MCP (KubeCon EU 2026): + · +- MCP tool annotations (hints, enforce server-side): +- MCP Elicitation (2025-06-18): diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f36024d --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +# Sith — Makefile +SHELL := /usr/bin/env bash + +BINARY := sith +PKG := github.com/ArdurAI/sith +CMD := ./cmd/sith +BIN_DIR := bin +GOLANGCI ?= golangci-lint + +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +LDFLAGS := -s -w \ + -X $(PKG)/internal/buildinfo.Version=$(VERSION) \ + -X $(PKG)/internal/buildinfo.Commit=$(COMMIT) \ + -X $(PKG)/internal/buildinfo.Date=$(DATE) + +.PHONY: all build test e2e lint fmt fmt-check vet tidy clean run ci help + +all: build + +build: ## Build the sith binary into bin/ + @mkdir -p $(BIN_DIR) + go build -trimpath -ldflags '$(LDFLAGS)' -o $(BIN_DIR)/$(BINARY) $(CMD) + +test: ## Run unit tests with the race detector and report coverage + go test -race -count=1 -coverprofile=coverage.out ./... + +e2e: ## Build and exercise the real binary as a subprocess + go test -race -count=1 -tags=e2e ./tests/e2e + +lint: ## Run golangci-lint (v2) + $(GOLANGCI) run ./... + +fmt: ## Format code (gofmt + goimports via golangci-lint v2 formatters) + $(GOLANGCI) fmt ./... + +fmt-check: ## Fail if formatting/imports would change anything + gofmt -l . | tee /dev/stderr | (! read) + $(GOLANGCI) fmt --diff ./... + +vet: ## Run go vet + go vet ./... + +tidy: ## Tidy and verify modules + go mod tidy + go mod verify + +clean: ## Remove build and coverage artifacts + rm -rf $(BIN_DIR) coverage.out + +run: build ## Build then run sith version + $(BIN_DIR)/$(BINARY) version + +ci: fmt-check vet lint test e2e build ## Run the full CI gate locally + +help: ## List targets + @grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | \ + awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-10s\033[0m %s\n",$$1,$$2}' diff --git a/README.md b/README.md index b6636e8..1280e1e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,30 @@ # Sith -**Status: planning.** No product code yet. +**Status: Slice 0 foundation.** The local-first CLI walking skeleton is runnable; Kubernetes +context discovery arrives in Slice 1. -Sith is a governed, multi-tenant control plane for **cross-cluster Kubernetes fleet -operations** — one place to safely *see* and *act* across many clusters, while deep -cluster access stays local. It is built on [Open Cluster Management (OCM)](https://open-cluster-management.io/). +Sith is ArdurAI's single-binary, local-first Kubernetes fleet tool: **k9s for your whole fleet**. +It is designed to aggregate every kubeconfig context without an account, telemetry, or cluster +data leaving the machine. The same source-abstract fleet model will later power an optional +governed hub. -The full charter, architecture, ADRs, threat model, roadmap, and competitive analysis -land via the initial planning pull request. The owner reviews the plan before any -implementation begins. +## Build and run -See the open PR and `docs/` for the plan. +Sith requires a supported Go 1.25 toolchain. + +```bash +make build +./bin/sith version +./bin/sith version --output json +./bin/sith clusters +``` + +Slice 0 intentionally returns a typed empty fleet through the stubbed `fleet.Source` seam. Run the +full local quality gate with a pinned golangci-lint v2.12.2 on `PATH`: + +```bash +make ci +``` + +The architecture, threat model, ADRs, and roadmap live under [`docs/`](docs/). Build-session +checkpoints are recorded under [`sessions/`](sessions/). diff --git a/cmd/sith/main.go b/cmd/sith/main.go new file mode 100644 index 0000000..7dd7414 --- /dev/null +++ b/cmd/sith/main.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package main is the process entrypoint for the Sith binary. +package main + +import ( + "os" + + "github.com/ArdurAI/sith/internal/cli" +) + +func main() { + os.Exit(cli.Execute()) +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..cff8e48 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,467 @@ +# Sith — Architecture + +**Status:** planning · **Date:** 2026-07-08 + +This document describes the intended architecture. It is a *plan*, not an as-built. Where +a decision has a dedicated record, the ADR is linked and is authoritative. + +--- + +## 1. One picture in words + +A central **hub** (the Sith control plane) maintains a governed, tenant-scoped model of a +fleet of Kubernetes clusters (**spokes**). It never holds cluster-admin credentials and +never reaches *into* a spoke on its own initiative. Instead, each spoke runs an +**outbound-only** OCM agent that dials the hub; the hub reaches cluster-local services +*through that reverse tunnel* (`cluster-proxy`) using **scoped, short-lived spoke tokens** +projected to the hub (`managed-serviceaccount`). + +On top of that OCM-brokered connectivity, Sith adds **three federations** — read, action, +policy — and exposes the whole thing as a **governed MCP server**. **Ardur** is the policy +decision point and identity broker for every action. + +**One binary, three run modes.** The same Go binary is the whole product. `sith` (CLI + +k9s-style TUI) and `sith ui` (local web "fleet IDE" on `localhost`) are the **day-0 local +mode**: they read the user's own kubeconfig contexts directly (client-side fan-out into an +informer/watch cache), with no hub, no OCM, no account, no telemetry — credentials never leave +the machine. `sith serve --mcp` exposes the same fleet as a **governed MCP server**. `sith hub` +is the **day-N federated mode** described above (multi-user, OCM-brokered reach to NAT'd/VPC'd +clusters, `Workspace` isolation, the governance pipeline). The **fleet model and the +enforcement pipeline are source-abstract**: a fleet fact's source is either a local kubeconfig +context (local mode) or an OCM-brokered spoke (hub mode), and everything above the source is +one code path. The rest of this document describes the hub; local mode is the same components +with the kubeconfig-direct source and single-user defaults. See +[`research/USE-CASE-AND-SHAPE.md`](research/USE-CASE-AND-SHAPE.md). + +## 2. Topology — outbound-only spokes, OCM-brokered reach + +```mermaid +flowchart TB + subgraph HUB["Sith Hub (control plane) — the crown jewel"] + API["Sith API / gateway"] + FM["Fleet model (read federation cache)"] + PEP["Intent PEP (policy enforcement point)"] + MCP["Governed MCP server"] + AUD["Audit-log (what-happened)"] + subgraph OCMHUB["OCM hub components"] + CPS["cluster-proxy: proxy servers"] + MSA["managed-serviceaccount: token store"] + end + end + + subgraph ARDUR["Ardur (PDP + identity broker + decision-ledger)"] + PDP["Policy decision point"] + IDB["Scoped-identity broker"] + LEDG["Decision-ledger (why-allowed)"] + end + + subgraph S1["Spoke cluster A (isolated VPC / behind NAT)"] + A_AGENT["OCM klusterlet + cluster-proxy AGENT (outbound-only)"] + A_SITH["Sith spoke agent (local allowlist + scoped identity)"] + A_SVC["cluster-local services: Argo CD · Grafana · Rollouts"] + end + + subgraph S2["Spoke cluster B (isolated VPC / behind NAT)"] + B_AGENT["OCM klusterlet + cluster-proxy AGENT (outbound-only)"] + B_SITH["Sith spoke agent (local allowlist + scoped identity)"] + B_SVC["cluster-local services: Argo CD · Grafana · Rollouts"] + end + + %% Outbound-only: tunnels are DIALED from spoke to hub + A_AGENT -- "outbound reverse tunnel (dials hub)" --> CPS + B_AGENT -- "outbound reverse tunnel (dials hub)" --> CPS + + %% Hub reaches spoke services THROUGH the established tunnel + CPS -. "reach cluster-local svc via tunnel" .-> A_SVC + CPS -. "reach cluster-local svc via tunnel" .-> B_SVC + + %% Scoped tokens projected spoke -> hub + A_AGENT -- "projects scoped SA token" --> MSA + B_AGENT -- "projects scoped SA token" --> MSA + + API --> FM + FM -- "reads via cluster-proxy + MSA token" --> CPS + API --> PEP + MCP --> PEP + PEP -- "may this intent run?" --> PDP + PEP -- "mint scoped exec identity" --> IDB + PDP --> LEDG + PEP --> AUD + + %% Action federation: dispatch signed intent down the SAME reverse channel + PEP -- "signed typed intent (via tunnel)" --> A_SITH + PEP -- "signed typed intent (via tunnel)" --> B_SITH + A_SITH -- "validate vs local allowlist, execute w/ local identity" --> A_SVC + B_SITH -- "validate vs local allowlist, execute w/ local identity" --> B_SVC +``` + +**The load-bearing property:** every arrow *from* a spoke is **outbound** (the spoke dials +the hub). The hub never needs inbound network access to a spoke, so spokes can live in +isolated VPCs / behind NAT. This is exactly what OCM `cluster-proxy` provides — verified: +*"The network proxy establishes reverse proxy tunnels from the managed cluster to the hub +cluster … enabling clients from the hub network to access services in the managed +clusters' network even when all the clusters are isolated in different VPCs."* +([cluster-proxy](https://github.com/open-cluster-management-io/cluster-proxy), v0.10.0, +2026-02-02). + +## 3. Component view — what the hub actually runs + +```mermaid +flowchart LR + subgraph CLIENTS["Clients"] + UI["Sith UI (operator console)"] + EXT["External agents: Claude Code · Codex · kagent"] + end + + subgraph CONTROL["Sith control plane"] + GW["API gateway
(authn: signed tokens, not headers)"] + MCPS["MCP server
(annotated tools + elicitation on writes)"] + RF["Read-federation service
(normalizes fleet model)"] + AF["Action-federation service
(typed-intent compiler + dispatcher)"] + PEP["PEP: enforcement pipeline"] + STORE[("Control-plane DB
tenant-scoped + RLS backstop")] + VAULT["Credential/key custody
(KMS envelope, per-tenant keys)"] + AUD["Audit-log"] + end + + subgraph OCM["OCM substrate (adopted)"] + CP["cluster-proxy"] + MSA["managed-serviceaccount"] + PLACE["ManagedCluster / Placement APIs"] + end + + ARDUR["Ardur PDP + identity broker + decision-ledger"] + + UI --> GW + EXT --> MCPS + GW --> RF + GW --> AF + MCPS --> RF + MCPS --> AF + RF --> PEP + AF --> PEP + PEP --> ARDUR + PEP --> STORE + PEP --> AUD + PEP --> VAULT + RF --> CP + RF --> PLACE + AF --> CP + PEP --> MSA +``` + +Notes: +- **Authn is from signed tokens, never from spoofable request headers.** (This is a direct + correction of a predecessor anti-pattern; see [ADR-0003](adr/0003-tenancy-isolation.md) + and [`THREAT-MODEL.md`](THREAT-MODEL.md).) +- **The MCP server and the UI share one enforcement pipeline (the PEP).** There is no + privileged back-door path; an external agent has *exactly* the governance a human does. +- **Read and action federation are separate services** with separate blast radii and + separate rate limits. Reads never require the write path; writes are the most hardened + surface in the system. + +## 4. The three federations + +### 4.1 Read federation + +- **The read source is abstracted.** A fleet fact comes from either a **local kubeconfig + context** (day-0 local mode, client-side) or an **OCM-brokered spoke** (day-N hub mode). The + fleet model and everything above it are identical across both; only the source adapter + differs. This is what makes the local client and the hub one product. +- Each Sith spoke agent (or the hub, via `cluster-proxy` + a `managed-serviceaccount` + token — or, in local mode, the kubeconfig-direct source) reports **inventory, health, + alerts, drift, image/CVE facts** for its cluster. +- The hub assembles a **tenant-scoped, cached, normalized fleet model**. Every record + carries a **freshness timestamp** and **source cluster**. +- **Cross-cluster correlation is a first-class query primitive**: "all clusters where + `payments` is Degraded", "clusters running image X with CVE Y", "fleet-wide drift vs + Git". Single-cluster tools cannot answer these; the federation can. +- **Least privilege by construction:** reads are bounded to what spoke agents choose to + report and to the scope of the projected `managed-serviceaccount` token — the hub cannot + read arbitrary cluster state just because it is the hub. +- **Staleness is surfaced, never hidden** — it is an input to abstention (§4.3). + +### 4.2 Action federation + +The write path is an **intent**, not a command: + +``` +Intent = { + id, tenant(workspace), actor, verb ∈ CLOSED_VOCAB, + targetSelector, # which clusters/objects — resolved against the fleet model + args, # typed, schema-validated per verb + justification, evidenceRefs, + signature # signed by the hub; verified by each spoke +} + +CLOSED_VOCAB (initial): + argocd.sync | argocd.rollback + rollout.promote | rollout.abort + deployment.scale | deployment.restart + gitops.open-pr + # PERMANENTLY EXCLUDED: exec, arbitrary apply, secret mutation, RBAC mutation +``` + +Flow (see also the enforcement pipeline, §6): + +```mermaid +sequenceDiagram + autonumber + participant C as Client (UI / MCP agent) + participant PEP as Sith PEP (hub) + participant PDP as Ardur PDP + participant IDB as Ardur identity broker + participant SP as Spoke agent (per cluster) + participant SVC as Cluster-local service + + C->>PEP: propose intent {verb, targetSelector, args, justification} + PEP->>PEP: authn (signed token) + tenant scope + role gate + verb-in-vocab + arg schema + PEP->>PDP: may {actor} run {verb} on {resolved targets} now? + PDP-->>PEP: allow / deny / require-approval(s) (+ record decision-ledger) + alt requires approval (e.g. prod) + PEP->>C: elicit typed approval (bound to arg hash) + C-->>PEP: approval (per-action, non-reusable) + end + PEP->>IDB: mint scoped, short-lived exec identity for this action + PEP->>SP: dispatch SIGNED intent over reverse tunnel (per target, wave-ordered) + SP->>SP: verify signature + validate vs LOCAL allowlist (independent) + SP->>SVC: execute with LOCAL scoped identity (dry-run first where applicable) + SVC-->>SP: result + SP-->>PEP: per-cluster outcome + PEP->>C: aggregated result + Note over PEP: audit-log every proposed + executed action (what-happened) +``` + +**Two independent blast-radius bounds:** the **closed verb vocabulary** at the hub *and* +the **spoke's own local allowlist + local RBAC**. A spoke **never blindly executes what +the hub sends** — it re-validates and uses its own identity. The safest first write is +`gitops.open-pr` (a proposal a human merges — zero new standing trust). See +[ADR-0004](adr/0004-typed-intent-action-model.md). + +### 4.3 Policy federation (the novel hard part) + +A single intent can touch N clusters, so per-action approval is not enough. The policy +layer (adjudicated by Ardur) reasons about **fan-out**: + +- **Environment gates.** `prod` never auto-executes; prod requires N-person approval; + a max-clusters-per-intent ceiling applies. +- **Wave / canary ordering.** Sith *plans* a rollout (dev → staging → 1 canary prod → + health-gate → rest); each wave is a **separately gated** step. No wave proceeds without + its own gate and a health check between waves. +- **Partial-failure semantics.** Stop-on-first-failure, auto-rollback of the failed wave, + and **idempotency/dedupe** so a retry cannot double-apply. +- **Federation-specific abstention.** If the fleet view is incomplete or stale, Sith + **refuses** fleet-wide action and says so honestly: *"37/40 clusters visible; 3 stale + >10m — I will not issue a fleet sync until they report."* Explicit "I won't act" is a + first-class, logged outcome, not an error. + +## 5. Data model (sketch) + +Not a schema — a conceptual model. `Workspace` is the tenancy anchor: all scoping is +"over clusters", not "one portal per cluster". + +```mermaid +erDiagram + WORKSPACE ||--o{ CLUSTER : "scopes" + WORKSPACE ||--o{ POLICY : "owns" + WORKSPACE ||--o{ INTENT : "issues" + WORKSPACE ||--o{ MEMBERSHIP : "grants" + CLUSTER ||--o{ FLEET_FACT : "reports" + INTENT ||--o{ INTENT_TARGET : "fans out to" + CLUSTER ||--o{ INTENT_TARGET : "is target of" + INTENT ||--o{ AUDITLOG : "produces" + POLICY ||--o{ DECISION : "yields" + INTENT ||--o{ DECISION : "adjudicated by" + + WORKSPACE { + id id PK + string name + string tenant_key "isolation anchor" + } + CLUSTER { + id id PK + id workspace_id FK + string ocm_managedcluster_ref "link to OCM object" + string labels "env, region, etc." + timestamp last_seen "freshness for abstention" + } + POLICY { + id id PK + id workspace_id FK + string version + json rules "env gates, approvals, caps, waves" + } + INTENT { + id id PK + id workspace_id FK + string actor "human or agent identity" + string verb "CLOSED_VOCAB only" + json target_selector + json args "schema-validated per verb" + string justification + string signature + string status "proposed|approved|dispatched|done|refused|failed" + } + INTENT_TARGET { + id id PK + id intent_id FK + id cluster_id FK + int wave "canary ordering" + string outcome "per-cluster result" + } + DECISION { + id id PK + id intent_id FK + string verdict "allow|deny|require-approval" + json reasons "why-allowed (Ardur decision-ledger)" + timestamp at + } + AUDITLOG { + id id PK + id intent_id FK + string phase "proposed|approved|dryrun|executed" + json detail "what-happened" + timestamp at + } + MEMBERSHIP { + id id PK + id workspace_id FK + string subject + string role "reader|operator|approver|admin" + } + FLEET_FACT { + id id PK + id cluster_id FK + string kind "inventory|health|alert|drift|cve" + json payload + timestamp observed_at "freshness" + } +``` + +Design points: +- **`Workspace` is the scoped tenancy object** — every query is scoped to a workspace, and + a **database-level backstop** (RLS) enforces it independently of application code (see + [ADR-0003](adr/0003-tenancy-isolation.md)). +- **`DECISION` (why-allowed, Ardur) and `AUDITLOG` (what-happened, Sith) are separate** + and together form a complete agent-action ledger. +- **`CLUSTER.last_seen` / `FLEET_FACT.observed_at`** are what abstention reads. + +## 6. Enforcement pipeline (the PEP) + +Every intent — from the UI *or* the MCP server — passes the **same** ordered gate. This is +the redesigned-clean descendant of the predecessor's action/exec broker pattern: + +``` +signed session/token → (authn; role/tenant from the signed claims, never headers) + ↓ +workspace membership → (is this actor a member of this workspace at all?) + ↓ +role gate → (reader | operator | approver | admin — least privilege) + ↓ +closed verb vocabulary → (fail-safe allowlist: unknown verb = refuse, not execute) + ↓ +arg schema validation → (typed per verb; reject anything not schema-valid) + ↓ +tenant scope resolve → (targetSelector resolved ONLY within this workspace's clusters) + ↓ +Ardur PDP query → (allow / deny / require-approval(s), fan-out aware) + ↓ +elicited approval → (per-action, non-reusable, bound to a hash of resolved args) + ↓ +scoped identity mint → (Ardur brokers short-lived per-action identity; ceiling < human) + ↓ +caps / budgets → (max clusters/intent, rate limits, token/action budgets) + ↓ +signed dispatch → (per target, wave-ordered; spoke re-validates independently) + ↓ +audit + decision ledger→ (proposed + approved + dry-run + executed, always) +``` + +**Fail-safe, not fail-open:** anything not explicitly permitted is refused. A CI test +asserts that *every* handler reaching a write path is classified against the closed +vocabulary — a forgotten classification fails the build, not production. (This inverts the +predecessor's fail-open "denylist of one".) + +## 7. MCP server surface + +Sith is exposed as an **MCP server** so external agents inherit the same governance. + +- **Read tools** carry `readOnlyHint: true` and hit the fleet model (no gate beyond + tenant scope). Examples: `fleet.inventory`, `fleet.health`, `fleet.correlate`, + `fleet.cve-search`. +- **Write tools** carry `destructiveHint: true` (and correct `idempotentHint`), map 1:1 to + the closed verb vocabulary, and require **Elicitation-based approval** (2025-06-18 MCP + primitive) bound to a hash of the resolved args. Examples: `intent.gitops-open-pr` + (first), later `intent.argocd-sync`, `intent.rollout-promote`, `intent.deployment-scale`. +- **Annotations are hints, so enforcement is server-side** — the MCP layer is a thin + client onto the same PEP (§6). Per the MCP guidance, hints are *not* trusted as + guarantees; the server enforces. + +See [ADR-0005](adr/0005-ai-mcp-ardur-pdp.md) for the reasoning and the "governed MCP +gateway to your whole fleet" positioning. + +## 8. Where Ardur plugs in + +[Ardur](https://github.com/ArdurAI/ardur) is ArdurAI's runtime-governance runtime. In +Sith it fills three roles, all at the PEP boundary: + +| Role | What it does | Complements | +|---|---|---| +| **PDP** | Adjudicates every intent (allow/deny/require-approval), fan-out aware | Sith PEP is the thin enforcement point | +| **Identity broker** | Mints short-lived, per-action scoped execution identity so the AI/agent never holds a cluster credential and its ceiling is strictly below the human's | OCM `managed-serviceaccount` for spoke-side scoping | +| **Decision-ledger** | Records *why* each action was allowed | Sith `AUDITLOG` records *what happened* | + +Sith is built with a **policy hook at the `executeIntent` boundary from day one** — even in +Phase 1 it returns "allow" for reads — so Ardur drops into that seam without re-architecture +when the write path arrives in Phase 2. + +## 9. Credential & key custody + +- The hub **does not store cluster-admin kubeconfigs**. Spoke reach uses OCM + `managed-serviceaccount` scoped tokens; action execution uses Ardur-brokered short-lived + identities validated locally by the spoke. +- Any secret the hub *must* hold (e.g. Git credentials for `gitops.open-pr`) is protected + with **envelope encryption via a KMS and per-tenant data keys** — never a single + process-wide key. See [ADR-0006](adr/0006-credential-key-custody.md). + +## 10. Explicit correction of predecessor anti-patterns + +This architecture deliberately avoids the failure modes found in a prior control-plane +prototype (documented, vendor-neutral, in [`THREAT-MODEL.md`](THREAT-MODEL.md) §7): + +- Authz from **signed claims**, never spoofable headers (no IDOR-by-header). +- A **real DB-level tenant backstop** (RLS) behind the app-layer scoping — not inert code. +- **Per-tenant key custody** via KMS envelope — not one env key that decrypts every tenant. +- **No shared admin cluster credential** in the center — scoped, local, per-action identity. +- **Fail-safe closed vocabulary** — not a fail-open denylist; **no shell, ever**. + +## 11. Reshape additions — local mode, connectors, custody, supply chain + +This section records the local-first dual-mode reshape (rationale in +[`research/USE-CASE-AND-SHAPE.md`](research/USE-CASE-AND-SHAPE.md)); the epics are E11–E13 in +[`EPICS.md`](EPICS.md). + +- **Four-mode connection/identity model.** (1) **Local (direct):** reads the user's kubeconfig + contexts on the machine; exec plugins run locally as kubectl does; any secret goes in the OS + keychain; nothing is uploaded. (2) **Federated (minion):** OCM klusterlet + cluster-proxy, + outbound-only; scoped `managed-serviceaccount` token; no admin kubeconfig in the hub. (3) + **Cloud IAM:** a thin per-cloud adapter enumerates clusters and mints **short-lived** tokens + (EKS get-token, AKS Entra+kubelogin, GKE plugin, ACK/CCE/TKE); no long-lived cloud keys at + rest. (4) **API key / JWT / OIDC:** for tool integrations and machine callers. One rule across + all four: authn from **signed token claims, never spoofable headers**; the agent identity + ceiling is strictly below the human's. Detail: + [`research/identity-connections-security.md`](research/identity-connections-security.md). +- **Connector framework (E12).** Out-of-process gRPC, SDK-first, **one canonical connector per + tool**, minor-additive versioning (the Grafana/Terraform pattern; the anti-pattern is + Backstage's in-process unversioned sprawl). Exactly **three connector kinds**: *read adapter* + (pull normalized facts), *brokered read-through* (deep-link to the tool's own UI — never + re-skin), *typed-action adapter* (map a closed verb to the tool's API). Day-1 six: Argo CD, + Flux, Helm, Prometheus, Loki, GitHub. +- **Cost read-overlay (E13).** Deploy/read OpenCost per cluster; aggregate at the hub into + per-workspace/team fleet rollups with GPU columns where DCGM exists. A read integration, not a + metering engine. +- **Supply chain + custody.** Releases are **cosign-signed** with **SLSA L2 provenance** and an + **SBOM** from the first tag (now cheap; scorecards weigh these before humans read code). + **SPIFFE IDs / mTLS are supported** as the identity model, but Sith does **not** require users + to run **SPIRE** (operationally heavy). Images are **multi-arch** (`amd64`+`arm64`) and + **registry-relocatable** for air-gap/China from day one (E9). diff --git a/docs/BUILD-SEQUENCE.md b/docs/BUILD-SEQUENCE.md new file mode 100644 index 0000000..302a3c8 --- /dev/null +++ b/docs/BUILD-SEQUENCE.md @@ -0,0 +1,391 @@ +# Sith — Phase-L Build Sequence (the adoption wedge) + +**Status:** locked · **Date:** 2026-07-10 · **Scope:** Phase L only (day-0 local fleet client + MCP read) + +This is the ordered slice plan for the **Phase-L wedge**: the single `sith` binary a DevOps +engineer runs locally — no account, kubeconfig-based, cache-first fleet view — growing into TUI + +local web UI + per-pod actions + MCP read tools. It is the reshape's centre of gravity +(`SITH-NOTION.md` §3, E11/#29). Federation/hub (E1–E10) is **deferred to phase-1+** and does not +gate any slice here. + +Each slice below states: **goal**, **mapped issue(s)**, **dependencies**, **acceptance criteria**, +**how it advances the "runnable + useful today" bar**, and **which open questions (Q12–Q15) it +touches**. Slices are built and merged in order; each ends green on CI and leaves the binary more +useful than the slice before. + +> **Ordering validated against the plan.** This sequence matches `SITH-NOTION.md` §3 ("build first" +> wedge list) and the E11 exit criteria (§E11): TUI/CLI cache-first render is the leanest day-0 wow; +> `sith ui` reuses the same fleet model; per-pod table stakes make the tool *complete*; the local +> source feeds the *same* E2 fleet model the hub will use (one code path above the source). **One +> deliberate divergence from the roadmap issue #39:** #39 lists E9 (#27) packaging under Phase L as a +> peer of E11. We fold only the *seed* of E9 (repo scaffold, CI, reproducible `make build`) into +> **Slice 0** and treat full brew/multi-arch/cosign/SLSA/SBOM release engineering (#27) as a +> **parallel track (Slice P)** that can run alongside Slices 1–6 without gating them — because the +> engineer's first `go install`/local build is enough to be "runnable today", and release packaging +> should harden once there is something worth shipping. This is noted for the owner, not assumed +> silently. + +--- + +## Who this is for — the target user's daily surface + +Sith is weighted toward what a working DevOps engineer touches **every day**, not toward a feature +matrix. The reference user (GR) works daily in: **Kubernetes**, **Helm**, **ArgoCD**, **Docker**, +**Python/bash**, **Fluentd/Fluent-bit**, **Grafana/Prometheus**, across **multi-cloud AWS/Azure/GCP**, +doing **vulnerability fixes** and **cloud networking**. The sequence below is ordered so the earliest +slices land the highest-frequency tasks: a fast multi-cluster fleet view, pod **logs/exec/describe**, +and cross-cluster search — then the common **observability + GitOps** integrations. + +**How the daily stack maps to the plan:** +- **Fleet view + logs/exec/describe/YAML** (the minute-to-minute loop) → Slices 1–3 (this phase). +- **Cross-cluster search + "where is X unhealthy / which clusters run image Y"** → Slice 2 (this phase). +- **Vuln fixes** (fleet-wide "which clusters run image X with CVE Y") → Slice 2's search over the + F2.4 CVE facts; deeper CVE ingestion matures with E2/F2.4. +- **ArgoCD / Grafana / Prometheus / Fluentd read overlays** (see sync state, dashboards, log + pipelines in-context) → **E12 connector framework (#30)** + **E13 cost overlay (#31)**, fast-follow + right after the wedge. The wedge deliberately ships the K8s-native loop first; these read + connectors layer on the same fleet model without disturbing Slices 0–6. +- **Multi-cloud AWS/Azure/GCP** → handled at the source layer: Slice 1's local-kubeconfig adapter + already honors each cloud's exec-credential plugin (aws/gcloud/az) locally, so a mixed-cloud + kubeconfig "just works" on day 0. +- **Governed GitOps writes** (ArgoCD sync, PR-open) are **out of Phase L** by design — read before + write (E4/E5, phase-2+). + +> **Hook — fuller GR-workflow profile (incoming).** A richer profile mined from GR's Notion + Claude +> references will be supplied next and folded in here. When it lands, capture it as +> `docs/GR-WORKFLOW-PROFILE.md` and refine (a) the per-slice "User-workflow fit" lines below and +> (b) the E12/E13 connector priority order to match the real frequency data. This section is the +> anchor for that update; do not block the current slices waiting on it. + +--- + +## The locked sequence at a glance + +| # | Slice | Mapped issues | Depends on | Hero outcome | +|---|---|---|---|---| +| **0** | Foundation walking-skeleton | E11 #29 (umbrella), E9 #27 (seed) | — | `sith version` / `sith clusters` run; typed empty fleet result through a stubbed source seam; CI green | +| **1** | Source-abstract fleet model + local-kubeconfig adapter + fan-out | **F2.1 #38** + **F11.1 #32** | Slice 0 | `sith clusters` lists **real** kubeconfig contexts; per-context read sessions; unreachable flagged | +| **2** | Cache-first fleet render (CLI + TUI) + cross-cluster search | **F11.2 #33** (+ F11.4 via #10/#38) | Slice 1 | `sith` TUI + `sith get … --all-clusters` render fleet from cache < 100 ms; one cross-cluster query | +| **3** | Per-pod table stakes | **F11.5 #35** | Slice 1 | logs / exec / port-forward / YAML per context with the user's own identity | +| **4** | Local web "fleet IDE" (`sith ui`) | **F11.3 #34** | Slices 1–2 | `sith ui` serves the same fleet model on loopback; embedded frontend | +| **5** | No-account / no-telemetry / keychain custody | **F11.6 #36** | Slice 0 (invariant), Slice 1 | keychain-backed secret custody; network-egress test proves no phone-home | +| **6** | MCP read tools (`sith serve --mcp`) | **F7.1 #37** | Slices 1(-2) | `fleet.inventory/health/correlate/cve-search` read tools over MCP, workspace-scoped, audited | +| **P** | Packaging & supply chain (parallel track) | **E9 #27** | Slice 0 | brew/multi-arch/cosign/SLSA/SBOM — hardens the install funnel; does not gate 1–6 | + +**Trust invariants (`CONVENTIONS.md` §7) hold from Slice 0** — loopback-only, no telemetry, +creds-never-leave, fail-safe. They are not a slice; they are enforced continuously and *verified* by +Slice 5. + +--- + +## Slice 0 — Foundation walking-skeleton + +**Goal.** Stand up the repo, module, CLI skeleton, config/logging, Makefile, and green CI, and prove +one end-to-end path: `sith version` prints build info, and `sith clusters` returns a **typed, empty** +`FleetResult` from a **stubbed source adapter** — the seam Slice 1 (F2.1) fills. This is the scaffold +everything else attaches to. + +**Mapped issues.** No feature issue of its own — it is the substrate under **E11 #29** and seeds +**E9 #27** (repo scaffold + CI + reproducible build). Fully specified in +[`specs/SLICE-0-foundation.md`](specs/SLICE-0-foundation.md). + +**Dependencies.** None. Branches off `dev`. + +**Acceptance criteria.** +- `go build ./...` succeeds; `make build` produces `bin/sith` with version metadata injected. +- `sith version` prints version / commit / build date / Go version (text and `--output json`). +- `sith clusters` calls the stubbed `fleet.Source`, gets an empty `FleetResult`, and prints a clean + "no clusters" result with exit 0. +- `sith ui` and `sith hub` are present as stubs that print a clear "not yet implemented — see + " notice and exit 0. +- CI is green: gofmt/format check, `go vet`, golangci-lint (v2, pinned), build, `go test -race`. +- `sessions/` exists with `README.md` + `JOURNAL-TEMPLATE.md`; the first session journal is + committed. + +**Advances the bar.** Turns an empty planning repo into a **runnable binary today**. Nothing useful +about the fleet yet, but the skeleton, the source seam, and the CI gate exist — so every later slice +is additive and always-green. + +**User-workflow fit.** Workflow-agnostic by design — this is pure scaffold. No GR daily task maps +here; the fit begins at Slice 1. + +**Open questions touched.** **None.** Slice 0 is deliberately independent of Q12–Q15. + +--- + +## Slice 1 — Source-abstract fleet model + local-kubeconfig adapter + client-side fan-out + +**Goal.** Implement the `fleet.Source` seam for real: a **local-kubeconfig** adapter that +enumerates every kubeconfig context, honors exec-credential plugins locally, opens an independent +non-blocking read session per reachable context, and streams normalized facts into the shared fleet +model (freshness + source stamped). This is simultaneously **F2.1** (the source-abstract model, +#38) and **F11.1** (kubeconfig auto-detect + fan-out, #32) — the local adapter *is* F11.1's fan-out. + +**Mapped issues.** **F2.1 #38** (source-abstract fleet model + local-kubeconfig adapter) + +**F11.1 #32** (kubeconfig auto-detect + client-side fan-out). Also lays the `Source` groundwork the +day-N OCM-spoke adapter (#9) reuses unchanged. + +**Dependencies.** Slice 0 (the `fleet.Source` interface + `sith clusters` wiring + model types). + +**Acceptance criteria** (from #38 / #32 / `SITH-NOTION.md` F2.1, F11.1): +- The fleet model is populated from local kubeconfig contexts with **no hub/OCM**. +- All contexts are detected; each reachable context streams reads; unreachable contexts are + **surfaced, never fatal** (independent, non-blocking per-context sessions). +- Exec-credential plugins (aws/gcloud/az helpers) run locally; **no credential or kubeconfig is + copied off the machine** (asserted by the Slice-5 egress test, seeded here). +- Every record is stamped with `observed_at` + `source cluster`; `last_seen` is maintained so a + cluster that stops reporting is detectable (F2.2/F2.5 shape). +- The same model code serves an OCM-spoke source unchanged (interface parity test — a second + in-memory adapter satisfies `fleet.Source` and flows through identical model code). +- `sith clusters` now lists real contexts with reachability + freshness. + +**Advances the bar.** First moment the tool is **useful**: run `sith clusters` and *see your whole +fleet* — every context, reachable or not — with zero config. This is the wedge's foundation; every +later surface renders this model. + +**User-workflow fit.** *"Morning fleet sweep."* GR opens the laptop with AWS EKS, Azure AKS, and +GCP GKE contexts in one kubeconfig; `sith clusters` enumerates all of them at once, running each +cloud's exec-credential plugin locally, and flags any context that's unreachable (expired SSO, +VPN down) — replacing a dozen `kubectl config use-context` + `get nodes` round-trips. + +**Open questions touched.** None blocking. (Q13 — local→hub upgrade UX — is the *seam* this model +enables, but it is a phase-1+ concern and does not gate Slice 1.) + +--- + +## Slice 2 — Cache-first fleet render (CLI + TUI) + cross-cluster search + +**Goal.** A k9s-style terminal view over the aggregated fleet that renders **from a local cache in +tens of milliseconds** (never spinner-first, never per-keystroke API round-trips), plus scriptable +CLI verbs (`sith get pods -A --all-clusters`) that return the *same* aggregated answers, plus the +wedge's signature cross-cluster query ("every context where `payments` is Degraded", "which contexts +run image X"). + +**Mapped issues.** **F11.2 #33** (cache-first fleet render, CLI + TUI). Folds in **F11.4** (local +cross-cluster search/correlation), which the roadmap routes through **#10** (correlation query) and +**#38** (source-abstract model) rather than a standalone issue. + +**Dependencies.** Slice 1 (the populated, cache-backed fleet model). + +**Acceptance criteria** (from #33 / `SITH-NOTION.md` F11.2, F11.4): +- The store is the **single render source**; the API is only a background sync target. Views and the + command bar (`:`/cmd-K fuzzy nav across all clusters) render **under ~100 ms** from cache; deltas + reconcile in the background without spinners. +- CLI verbs (`--all-clusters`) return the same aggregated answers as the TUI (parity test). +- One cross-cluster query returns a **correct** answer over **≥ 2 contexts**; any stale/unreachable + context is **flagged** in the result (coverage never silently dropped — reuses F2.5 staleness). + +**Advances the bar.** This is the "**k9s for your whole fleet**" wow. The tool now does the one thing +no OSS tool ships (cross-cluster read + correlation) and does it fast. This is the demo that earns +adoption. + +**User-workflow fit.** *"Incident triage across the fleet."* A Prometheus alert fires for +`payments`. Instead of hopping clusters, GR runs `sith` and asks once — "every context where +`payments` is Degraded" — or `sith get pods -A --all-clusters | grep CrashLoopBackOff` from an SSH +box, and gets a fast, cache-first answer with stale clusters flagged. Same query answers the vuln +sweep: "which contexts run image `X`" ahead of a CVE patch (F2.4 facts). + +**Open questions touched.** **Q12 (local-mode hero surface).** Default locked here: **TUI/CLI first** +(the leanest day-0 wow), `sith ui` as the fast-follow (Slice 4). Rationale: the TUI and web UI render +the *same* fleet model, so building the cache-first render + query engine first makes the web UI a +thin second surface rather than a parallel effort. If the owner overrides to "web-first" or "both +together", Slices 2 and 4 swap/merge — the model layer (Slice 1) is unchanged either way. + +--- + +## Slice 3 — Per-pod table stakes (logs / exec / port-forward / YAML) + +**Goal.** The commodity single-cluster operations whose *absence* drove the Lens exodus — logs, +exec, port-forward, YAML view/edit — run as ordinary K8s API calls against the selected context with +the **user's own kubeconfig identity**. Present so the local tool is *complete*, explicitly **not** +governed typed intents and carrying no fleet-action semantics. + +**Mapped issues.** **F11.5 #35** (per-pod table stakes). + +**Dependencies.** Slice 1 (per-context client sessions). Independent of Slice 2 (can build in +parallel once Slice 1 lands, but sequenced after it for a clean trunk). + +**Acceptance criteria** (from #35 / `SITH-NOTION.md` F11.5): +- Logs (stream + tail), exec (interactive shell into a pod), port-forward, and YAML view/edit work + per context in local mode, using the user's own identity. +- These paths are **clearly local conveniences**, distinct from the governed action model: they are + never dispatched as typed intents; the closed vocabulary + no-shell rule still bind every + *governed* (hub/agent) path (a code-level boundary, asserted by test — local exec must not route + through any intent/PEP path). + +**Advances the bar.** Removes the "but it can't even tail logs" objection. After this slice a Lens/k9s +user can *fully replace* their per-cluster tool with `sith` and additionally get the fleet view. + +**User-workflow fit.** *"Debug the failing pod."* Once GR spots the CrashLoop in the fleet view, +the next reflex is `logs -f`, `exec -it` for a quick `curl`/`nslookup` (cloud-networking checks), +`describe`/YAML to see the events and the mounted config, and `port-forward` to hit a Grafana or a +service locally. This slice makes those work per context with GR's own identity — the exact k9s/Lens +loop, now available across the whole fleet without switching tools. + +**Open questions touched.** None. (The local-vs-governed distinction is settled by `SITH-NOTION.md` +§6 guardrails, not an open question.) + +--- + +## Slice 4 — Local web "fleet IDE" (`sith ui`) + +**Goal.** The same source-abstract fleet model served as a local web UI on **loopback only** — the +visual "Lens-but-better" surface — from the same binary's **embedded** frontend. Single-user, +kubeconfig-direct, no account, no telemetry. Reuses the *same* frontend the hub console (E8) will +serve; here it runs in local mode. + +**Mapped issues.** **F11.3 #34** (local web fleet IDE `sith ui`). + +**Dependencies.** Slice 1 (model) + Slice 2 (the render/query patterns the web UI mirrors; per-pod +ops from Slice 3 surface here too). + +**Acceptance criteria** (from #34 / `SITH-NOTION.md` F11.3): +- `sith ui` serves the aggregated fleet view (multi-cluster views + fleet search/correlation + + per-pod table stakes) on `localhost`, **binds loopback only** (external bind is refused — invariant + test), with **no account and no telemetry**. +- It reuses the **same frontend** as the hub console (one codebase, two modes) — the frontend is a + client of the fleet model with no privileged path (ADR-0002). +- The frontend is embedded in the Go binary (`//go:embed`), so `sith ui` needs no separate asset + install. + +**Advances the bar.** Gives the visual audience (Lens refugees who want a GUI, not a TUI) a reason to +adopt, without a second install or an account wall. Same engine, second face. + +**User-workflow fit.** *"Share a view / prefer a GUI."* When GR wants a visual surface — scanning +many namespaces' health at a glance, reading a long YAML, or showing a teammate the fleet during an +incident call — `sith ui` opens the same fleet model in the browser on loopback, no account, no +second install. It's the Lens/Headlamp GUI habit, kept local and multi-cluster. + +**Open questions touched.** **Q12** (see Slice 2 — this slice is the "web" arm of the hero decision). +If the owner picks web-first, this slice moves ahead of Slice 2's TUI work; the model layer is shared +regardless. + +--- + +## Slice 5 — No-account / no-telemetry / keychain custody + +**Goal.** Make the trust promises **provable**: no login wall, no phone-home, and any local secret +kept in the **OS keychain** (osxkeychain / wincred / secret-service), with a **fail-loud** fallback +(fail or encrypt-at-rest, **never** silent plaintext). The no-account/no-telemetry posture is a +Slice-0 *invariant*; this slice implements the keychain *mechanism* and the *proof* (an egress test). + +**Mapped issues.** **F11.6 #36** (no-account / no-telemetry / keychain custody). + +**Dependencies.** Slice 0 (invariant enforced from the start) + Slice 1 (there is now a client that +*could* hold a secret). Naturally sequenced just before Slice 6, which is the first feature that +persists a secret (the local MCP token, Q14). + +**Acceptance criteria** (from #36 / `SITH-NOTION.md` F11.6): +- **No account** and **no network telemetry** in local mode, **verified by an automated network + check** (a test that fails if the binary opens any egress connection during a local-mode session + except to the user's own clusters). +- Any persisted secret goes to the OS keychain; a missing/unavailable keychain **fails loudly or + encrypts at rest** — never silent plaintext (the gh-CLI mistake). Fallback behavior is asserted by + test. +- Kubeconfig credentials are read in place, never copied or uploaded (shares the Slice-1 assertion). + +**Advances the bar.** Converts "trust us" into "here's the test that proves it". This is the exact +promise that wins the Lens-refugee audience; making it *verifiable* is the differentiator. + +**User-workflow fit.** *"Run it on a work laptop without a second thought."* GR points `sith` at +production kubeconfigs holding cloud IAM exec creds; the trust posture — no account, no phone-home +(provable by the egress test), secrets in the OS keychain, creds read in place — is what makes that +safe on a corp machine under security review. It removes the objection before it's raised. + +**Open questions touched.** **Q15 (local-mode telemetry stance).** Default locked here: **permanent +hard no** — no telemetry, not even off-by-default opt-in, in Phase L (the Lens backlash argues for +it; `SITH-NOTION.md` Q15). If the owner later wants an explicit, disclosed opt-in, it is a separate +ADR-gated decision, not a silent addition. **Q14** is set up here (keychain is where a local MCP token +would live) and consumed by Slice 6. + +--- + +## Slice 6 — MCP read tools (`sith serve --mcp`) + +**Goal.** Expose the fleet model to external agents (Claude Code / Codex / Cursor) as **read** tools +carrying `readOnlyHint: true` — `fleet.inventory`, `fleet.health`, `fleet.correlate`, +`fleet.cve-search` — hitting the same fleet model, scoped to the caller's workspace, audited like any +other read. In local single-user mode there is one workspace (the machine); the tenant-scoping code +path is the *same* one the hub uses (no privileged MCP data path). + +**Mapped issues.** **F7.1 #37** (MCP read tools). + +**Dependencies.** Slice 1 (model) and ideally Slice 2 (correlation/CVE query engine backs +`fleet.correlate` / `fleet.cve-search`). + +**Acceptance criteria** (from #37 / `SITH-NOTION.md` F7.1): +- The four read tools return workspace-scoped fleet answers (including correlation and CVE search), + carry `readOnlyHint: true`, and are audited. +- MCP reads go through the **same** scope-resolution path as the CLI/UI — the MCP layer has **no + privileged data path** (a test asserts an MCP read cannot see beyond the caller's scope). +- `sith serve --mcp` binds **loopback only** in local mode (invariant). + +**Advances the bar.** Makes Sith useful to the **AI-native engineer** — the fastest-growing adoption +vector. An agent can now answer "which of my clusters run image X with CVE Y?" through governed, +read-only, audited tools, with the exact same enforcement the hub will apply to writes later. + +**User-workflow fit.** *"Ask the agent about the fleet."* GR works in Claude Code / Codex daily; +with `sith serve --mcp` the agent can answer "which clusters run image `X` with CVE `Y`?" or "where +is `payments` unhealthy?" through governed, read-only, audited tools — turning the fleet model into +something the AI in GR's editor can query, without ever handing it a cluster credential or a shell. + +**Open questions touched.** **Q14 (local MCP auth).** Default locked here: **loopback trust + +optional short-lived local token held in the OS keychain** (Slice 5). Rationale: in single-user local +mode the loopback boundary is the trust boundary; an optional keychain-held token defends against +shadow-MCP local clients without a login wall. The hub-mode signed-token model (E1) is a separate, +later path. If the owner wants per-agent registered identities even locally, that is a superset +built on this seam. + +--- + +## Slice P — Packaging & supply chain (parallel track, E9 #27) + +**Goal.** The install funnel: single binary via `brew`/package managers, multi-arch +(`linux/amd64`+`arm64`, `darwin/arm64`), registry-relocatable references, and **cosign-signed +releases + SLSA L2 provenance + SBOM from the first tag**. + +**Mapped issues.** **E9 #27** (deployment & packaging — the local-client single-binary funnel). + +**Dependencies.** Slice 0 (a buildable binary + CI). **Runs in parallel with Slices 1–6 and gates +none of them** — the divergence from roadmap #39 noted at the top. It becomes *required* before any +public `brew install sith` announcement, not before the binary is useful locally. + +**Acceptance criteria** (subset relevant to Phase L, from #27 / `SITH-NOTION.md` E9): +- Reproducible multi-arch release build (goreleaser or equivalent) producing signed binaries. +- cosign signature + SLSA provenance attestation + SBOM attached to each tag. +- A working `brew` formula (tap) installs the binary; `sith version` reports the release metadata. + +**Advances the bar.** Turns "runnable today from source" into "`brew install sith && sith` in under a +minute" — the actual adoption on-ramp (`SITH-NOTION.md` §2). Deferred just enough that we harden the +supply chain once there is something worth shipping. + +**User-workflow fit.** *"Install it like any other CLI tool."* `brew install sith` (or the distro +package) is the same muscle memory as installing `kubectl`, `helm`, `k9s`, or `argocd` — the +frictionless on-ramp that gets Sith onto GR's machine and, later, teammates'. + +**Open questions touched.** None in Phase L (Q3 KMS/HSM reference is a hub/heavy-profile concern, +not the local single-binary funnel). + +--- + +## Open-question dependency summary (Q12–Q15) + +| Q | Question (`SITH-NOTION.md` §9) | Slice(s) it touches | Default locked (overridable by owner) | +|---|---|---|---| +| **Q12** | Local-mode hero surface: TUI vs `sith ui` vs both | Slices 2 & 4 | **TUI/CLI first, `sith ui` fast-follow.** Shared model makes either order cheap. | +| **Q13** | Local→hub upgrade UX (graduate a kubeconfig cluster to an OCM minion) | none in Phase L (seam only) | **Deferred to phase-1+.** Slice 1's source-abstract model is the seam; no Phase-L slice blocks on it. | +| **Q14** | Local MCP auth (loopback trust / short-lived token / keychain secret) | Slice 6 (set up by Slice 5) | **Loopback trust + optional short-lived keychain-held token.** | +| **Q15** | Local-mode telemetry stance (permanent no vs later opt-in) | Slice 5 (+ Slice-0 invariant) | **Permanent hard no in Phase L.** Any future opt-in is ADR-gated, disclosed, off-by-default. | + +None of Q12–Q15 block **Slice 0**. Q12/Q14/Q15 have safe defaults recorded above and can proceed +without owner input; the owner can override at any point and only the named slices shift. + +--- + +## Sequencing discipline (never violated) + +Read before write. The whole Phase-L wedge is **read-only** — there is no governed write path in any +slice here; per-pod ops (Slice 3) are the user's own identity, not Sith-brokered actions. The first +governed write (`gitops.open-pr`, E4/#22) is **P2**, after the hub exists. `exec` as a *governed* +action is never expressible (`SITH-NOTION.md` §6). This ordering keeps the wedge honest: Sith earns +adoption as the tool you use to *see* your fleet, before it ever asks to *act* on it. diff --git a/docs/CHARTER.md b/docs/CHARTER.md new file mode 100644 index 0000000..fe091c0 --- /dev/null +++ b/docs/CHARTER.md @@ -0,0 +1,180 @@ +# Sith — Charter + +**Status:** planning · **Date:** 2026-07-08 · **License:** Apache-2.0 + +This charter states *why Sith exists*, *who it is for*, *what specifically it owns*, and +*how we will know it worked*. It is deliberately narrow. The single greatest risk to a +project in this space is scope drift into territory that larger, better-funded, or +CNCF-blessed efforts already own; the charter's job is to hold the line. + +--- + +## 1. The problem + +An organization runs **many** Kubernetes clusters — across teams, regions, cloud +accounts, VPCs, and network boundaries. Each cluster already has excellent *local* +operational tooling: Argo CD, Argo Rollouts, Prometheus/Grafana, its own RBAC. What is +missing is a **governed place to operate across the fleet**: + +- **Seeing across clusters is hard.** Answering "which of my clusters have `payments` + Degraded right now?" or "which clusters run image X with CVE Y?" means logging into N + consoles or building a bespoke aggregator. Single-cluster tools structurally cannot + answer fleet-wide questions. +- **Acting across clusters is dangerous.** The moment you build a central place that can + *act* on many clusters, you have built the highest-value attack surface and the largest + blast radius in the estate. Most teams either (a) don't build it and operate by hand, or + (b) build it without the governance to make it safe. +- **Deep cluster access does not belong in the center.** Centralizing cluster-admin + kubeconfigs so a hub can act is exactly the anti-pattern that turns one compromise into + a fleet-wide breach. Yet "reach into the cluster from the center" is the naive design + everyone reaches for. +- **AI raises the stakes.** Teams now want agents (their own, and third-party agents like + Claude Code / Codex / kagent) to operate infrastructure. An agent on top of an + ungoverned control plane is a loaded gun pointed at the fleet. + +The gap is therefore **not** "another multi-cluster tool." It is a **governed federation +of *see* and *act* that keeps deep access local and makes every cross-cluster action +policy-gated, scoped, signed, and audited** — including when the actor is an AI agent. + +## 2. The thesis + +Four claims, each independently defensible, together define Sith: + +1. **The transport is commodity — adopt it, do not build it.** An outbound-only + per-cluster agent plus a central hub that reaches cluster-local services is *already* + shipped, hardened, and maintained by CNCF and vendors. Building it bespoke re-invents + security-sensitive infrastructure worse than the incumbents. + → **Decision: build on Open Cluster Management (OCM).** See + [ADR-0001](adr/0001-adopt-ocm-vs-bespoke-tunnel.md). +2. **The differentiator is governance of *action federation*, not visibility.** Read + federation is necessary and is the first vertical, but it is increasingly commoditized. + The durable, hard, valuable thing is **safely fanning a *typed intent* out to N + clusters under policy** — with environment gates, wave ordering, partial-failure + semantics, and honest abstention. This is a *policy* problem, and policy over + distributed action is genuinely under-served as an adoptable primitive. +3. **AI is a client of the governance, not a bypass of it.** Sith exposes the fleet as a + **governed MCP server**. Its own agent and any external agent go through the *same* + policy decision point, the *same* closed verb vocabulary, the *same* scoped identity + broker, the *same* audit + decision ledger. The AI never holds a cluster credential and + never gets a shell. +4. **Adoption is local-first; governance is the moat, not the on-ramp.** What an engineer + installs on day 0 is a **single binary that shows their whole fleet from the kubeconfigs + already on their machine** — a k9s-style CLI/TUI (plus an optional local web "fleet IDE"), + no account, no telemetry, no hub, no agents. The governed hub is the *same binary* run as a + control plane when a team outgrows kubeconfig fan-out (clusters behind NAT/VPCs, shared + audit, multi-approver prod). You earn the right to govern a fleet by first being the tool + the engineer already uses to see it. See + [`docs/research/USE-CASE-AND-SHAPE.md`](research/USE-CASE-AND-SHAPE.md). + +## 3. Target user + +**Top-of-funnel (day 0):** the **individual DevOps / SRE / platform engineer** juggling +several clusters from their laptop who wants one fast local view across all of them — no +account, no server, no telemetry. This is the adoption wedge; it is how Sith gets installed. + +**Primary buyer (day N):** the **platform / SRE / fleet-operations engineer** at an +organization running **tens to hundreds** of Kubernetes clusters who is accountable for *safe* +cross-cluster operations and for *who did what, where, and why*. + +**Secondary:** +- **Security / compliance owners** who need a defensible answer to "prove what your + operators — human and agent — are allowed to do and did do across the fleet." +- **AI-forward platform teams** who want to let agents operate infrastructure but only + behind hard governance. + +**Explicitly not the target:** application developers wanting a self-service catalog +(that is an IDP). Note the nuance: Sith does **not** build *another single-cluster* console +(Headlamp/k9s/Lens serve that well), but the **aggregated multi-cluster** local view *is* +ours and is the day-0 on-ramp (see [`SCOPE.md`](SCOPE.md)). + +## 4. The wedge (what Sith owns) + +Sith has **two wedges**, and holding both is the strategy (conflating them is what killed the +predecessor): + +**(A) The adoption wedge — the local aggregated fleet client.** A single binary that renders +every kubeconfig context on the engineer's machine as one searchable fleet — a k9s-style +CLI/TUI plus an optional local web "fleet IDE" — with cross-cluster correlation single-cluster +tools cannot do. No account, no telemetry, no hub, no agents. This is how Sith gets *installed* +(the empty OSS slot: k9s is one-context-at-a-time, Headlamp is per-cluster-centric, Lens has an +account wall, and the only aggregated client, Aptakube, is closed and paid). + +**(B) The durable wedge — governed action federation (the moat).** The **governed access + +action federation layer**, expressed as three federations over the *same fleet model* and, in +day-N hub mode, over OCM-brokered connectivity: + +- **Read federation.** A tenant-scoped, normalized **fleet model** (inventory, health, + alerts, drift) assembled from OCM-brokered reads, with **cross-cluster correlation** as + a first-class primitive. +- **Action federation.** Writes are **typed intents** from a **closed verb vocabulary** + (`argocd.sync|rollback`, `rollout.promote|abort`, `deployment.scale|restart`, + `gitops.open-pr`). Intents are **signed**; each spoke validates against a **local + allowlist** and executes with its **own scoped identity**. No shell, no free-form + `apply`, no secret/RBAC mutation. First write shipped = `gitops.open-pr`. +- **Policy federation.** Fan-out reasoning: environment gates, wave/canary ordering with a + per-wave gate, partial-failure/auto-rollback, idempotency, and **federation-specific + abstention** when the fleet view is incomplete or stale. + +**Ardur is the PDP + identity broker + decision-ledger.** Sith's enforcement points ask +Ardur *"may this actor issue this intent across these clusters now?"* → allow / deny / +require-approval(s). See [ADR-0005](adr/0005-ai-mcp-ardur-pdp.md). + +## 5. Why now + +Verified against the July-2026 landscape (see [`COMPETITIVE.md`](../COMPETITIVE.md) for +citations): + +- **The substrate matured.** OCM's `cluster-proxy` and `managed-serviceaccount` addons + reached **v0.10.0 (Feb 2026)** and deliver the exact outbound-only, cross-VPC, + reach-cluster-local-services design as adoptable addons — so the year we would have + spent building transport is now spent on governance instead. +- **AI operations went mainstream — and so did the danger.** Komodor shipped an + extensible multi-agent architecture (Mar 2026) with MCP/OpenAPI bring-your-own-tools; + SUSE Rancher Prime shipped an agentic "crew" + external MCP at KubeCon EU 2026. The + market has validated *agents operating fleets*; what remains under-served is + **vendor-neutral, OSS, governed action federation** as a primitive rather than a feature + of one vendor's platform. +- **MCP gained the safety primitives to build on.** The 2025-03-26 spec added tool + annotations (`readOnlyHint`/`destructiveHint`/…, explicitly *hints, enforce + server-side*), and the 2025-06-18 spec added **Elicitation** (a server can request + structured user input/approval mid-flow) — the native shape for human-in-the-loop + approval gates. + +## 6. Success criteria + +Success is defined by **falsification-first** discipline: each milestone must be able to +*disprove* an assumption cheaply before the next is funded. + +- **M0 (falsification).** We can reach a spoke's in-cluster service (Grafana / Argo CD) + from an OCM hub via `cluster-proxy` + `managed-serviceaccount` in **≤ 1 day of setup**. + *If yes → the "build the transport" scope is deleted.* *If no → the whole premise is + wrong and we stop before writing product code.* +- **Adoption (local mode, day 0).** A new user goes from `brew install sith` to a populated, + searchable cross-cluster answer over their own kubeconfig contexts in **under 10 minutes**, + offline, with **nothing leaving their machine** — no account, no server, no telemetry. The + same read surface is also exposed as annotated **MCP read tools** so an AI agent inherits it. +- **P1 (read federation).** From one place, correctly answer a cross-cluster question + ("every cluster where deployment X is unhealthy") across **≥ 2 spokes**, tenant-scoped, + with staleness surfaced per cluster. +- **P2 (first governed write).** A `gitops.open-pr` intent flows end-to-end through the + Ardur PDP and opens a real PR on a target repo — **with a complete decision-ledger + + audit-log entry** and *zero* cluster credentials ever reaching the center or the AI. +- **P3 (policy federation + MCP).** A wave-ordered, approval-gated intent fans out across + spokes with a gate per wave and correct **abstention** when a spoke is stale; the same + governance applies identically to an external MCP client. + +**Non-criteria (we will resist measuring these):** number of integrations, breadth of +read surface, UI polish, agent autonomy level. Depth of *governance correctness* beats +breadth of *features* for this product. + +## 7. Guardrails (anti-drift contract) + +1. If a capability is available as a maintained OCM addon or upstream project, **adopt it** + rather than building it. +2. If a proposed feature belongs to "developer portal", "GitOps controller", "scheduler", + or "telemetry lake", it is **out of scope** — full stop (see [`docs/SCOPE.md`](SCOPE.md)). +3. The write path may only ever grow **typed verbs in a reviewed closed vocabulary**. + Adding a verb is an ADR-level decision. `exec` and free-form `apply` are permanently + excluded. +4. Multi-tenant isolation, signed intents, per-spoke local enforcement, and scoped + identity are **day-one requirements**, not later hardening. diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md new file mode 100644 index 0000000..2d4b306 --- /dev/null +++ b/docs/CONVENTIONS.md @@ -0,0 +1,328 @@ +# Sith — Engineering Conventions + +**Status:** locked for Phase L · **Date:** 2026-07-10 · **Audience:** every build session (human or agent) + +This document is the contract every Sith build session follows. It is deliberately terse and +prescriptive: a fresh builder session should be able to read this once and produce commits, tests, +and CI that pass on the first try. If a rule here conflicts with a habit, the rule wins. + +Product identity is **ArdurAI**. Never `radiantic`, never any predecessor name, anywhere — code, +comments, commits, docs, module paths, or artifacts. + +Related: [`ARCHITECTURE.md`](ARCHITECTURE.md), [`adr/0002-stack-and-language.md`](adr/0002-stack-and-language.md), +[`BUILD-SEQUENCE.md`](BUILD-SEQUENCE.md), [`specs/SLICE-0-foundation.md`](specs/SLICE-0-foundation.md). + +--- + +## 1. Branch model + +| Branch | Role | Rules | +|---|---|---| +| `main` | **Release.** Seed + tagged releases only. | Protected. No direct pushes. Only receives merges from `dev` at release time. Tags (`vX.Y.Z`) are cut here. **Do not touch `main` during Phase L.** | +| `dev` | **Integration.** The default PR target and the trunk all feature work merges into. | Protected. All feature PRs target `dev`. Must stay green. | +| `feat/*`, `fix/*`, `docs/*`, `chore/*`, `refactor/*`, `test/*`, `ci/*`, `build/*` | **Feature branches.** One slice or one coherent change each. | Branched **off `dev`**. PR **into `dev`**. Deleted after merge. | + +**Naming.** `feat/-` — e.g. `feat/slice-0-foundation`, +`feat/fleet-source-adapter`, `feat/mcp-read-tools`. Keep slugs kebab-case and specific enough to +identify the slice from `BUILD-SEQUENCE.md`. + +**Flow.** +1. `git checkout -b feat/ dev` +2. Commit in small, signed, conventional increments (§2). +3. Before opening/refreshing the PR: `git fetch origin && git rebase origin/dev` (keep the branch + current; resolve conflicts locally). Rebase — do **not** merge `dev` back into the feature branch. +4. Open a PR into `dev`. CI must be green (§5). +5. Merge with a **merge commit** (`--no-ff`) or **rebase-merge** — never **squash-merge**. Squash + collapses the per-commit `Signed-off-by` and SSH signatures into one re-authored commit and + breaks the DCO + signature chain. Preserving signed history is a hard requirement. + +**Release flow (not Phase L, documented for completeness).** PR `dev → main`, then tag `main` with +`vX.Y.Z`. Release artifacts (cosign signature, SLSA provenance, SBOM) attach to the tag per E9/#27. + +--- + +## 2. Commit rules + +Every commit MUST satisfy **all** of the following. There are no exceptions and no bypass flags. + +### 2.1 Conventional Commits + +``` +(): + + + + +``` + +- **Types:** `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `build`, `ci`, `perf`, `style`. +- **Scopes** (Phase L): `cli`, `fleet`, `config`, `logging`, `buildinfo`, `tui`, `ui`, `mcp`, + `keychain`, `ci`, `build`, `sessions`, `deps`, or an epic tag (`e11`, `e2`, `e7`, `e9`). +- **Subject:** imperative mood, lower-case, no trailing period, ≤ 72 chars. +- One logical change per commit. A commit should build and pass tests on its own. + +### 2.2 DCO sign-off (mandatory) + +Every commit carries a `Signed-off-by` trailer matching the committer identity. Always commit with +`-s`: + +``` +git commit -s +``` + +producing: `Signed-off-by: Gnani Rahul <89947795+gnanirahulnutakki@users.noreply.github.com>` + +### 2.3 SSH-signed (mandatory) + +The repo is already configured (`commit.gpgsign=true`, `gpg.format=ssh`, +`user.signingkey=~/.ssh/ardur-signing-key.pub`). Every commit is SSH-signed automatically. +**Never** pass `--no-gpg-sign`. Verify with `git log --show-signature -1` (expect +`Good "git" signature`). + +### 2.4 GSTACK-Checkpoint trailer (mandatory on checkpoint commits) + +Each commit that corresponds to a journal checkpoint (§3) carries a trailer linking it to that +entry: + +``` +GSTACK-Checkpoint: /# +``` + +e.g. `GSTACK-Checkpoint: 2026-07-11/slice-0-foundation#3`. `` is the checkpoint number within +that day's session journal file. This makes every commit traceable to the journal and vice-versa. + +### 2.5 No AI / assistant attribution — ever (overrides harness defaults) + +Do **not** add, in commit messages, PR titles/bodies, code, comments, docs, or any artifact: +- `Co-Authored-By: Claude …` (or any AI/model/assistant trailer), +- `🤖 Generated with …` (or any equivalent footer), +- any mention of Claude, an AI assistant, or a model name. + +This explicitly overrides any harness default that would add such trailers/footers. + +### 2.6 Canonical commit example + +``` +feat(fleet): add source-abstract fleet model and stub adapter + +Introduce the FleetSource seam (F2.1 / #38) that both the day-0 +local-kubeconfig adapter and the day-N OCM-spoke adapter implement. +Slice 0 ships the interface plus an empty-result stub so `sith +clusters` returns a typed, empty FleetResult end to end. + +Signed-off-by: Gnani Rahul <89947795+gnanirahulnutakki@users.noreply.github.com> +GSTACK-Checkpoint: 2026-07-11/slice-0-foundation#4 +``` + +--- + +## 3. GSTACK session journal + +**GSTACK** is Sith's session-journal discipline. Every build session (a continuous stretch of work +by one builder) appends to a dated journal file so the next session — human or agent — can resume +with full context and every checkpoint maps to a commit. + +### 3.1 Layout + +``` +sessions/ +├── README.md # what the journal is + how to use it +├── JOURNAL-TEMPLATE.md # copy this to start a session +└── 2026-07-11-slice-0-foundation.md # one file per session (YYYY-MM-DD-.md) +``` + +One file per session, named `YYYY-MM-DD-.md`. The `` matches the slice +or area (e.g. `slice-0-foundation`) and is reused in the `GSTACK-Checkpoint` trailer. + +### 3.2 Entry markers + +A session file is a chronological log of five entry types. Each entry is one line prefixed by its +marker; a **checkpoint** groups the preceding G/S/A/T into a numbered milestone. + +| Marker | Name | Records | +|---|---|---| +| `[G]` | **Goal** | The objective of this work unit — usually a slice or a sub-task, with its issue number(s). | +| `[S]` | **Scope** | Files/packages in play; what is explicitly *out* of scope for this unit. | +| `[A]` | **Action** | What was actually done — changes made, commands run, decisions taken. | +| `[T]` | **Test** | How it was verified — tests added/run, CI result, manual/e2e checks, observed output. | +| `[C]` | **Checkpoint** | A numbered milestone: the commit SHA(s), the decision recorded, and the next step. The `#` here is the trailer's ``. | + +The journal is the **stack** of these G/S/A/T/C entries — hence *GSTACK*. + +### 3.3 Rules + +- Start every session by copying `JOURNAL-TEMPLATE.md` to `sessions/YYYY-MM-DD-.md`. +- Append entries as you work — do not reconstruct them at the end. +- Every `[C]` checkpoint corresponds to exactly one commit carrying the matching + `GSTACK-Checkpoint: YYYY-MM-DD/#` trailer. +- Record open questions and decisions (including which of Q12–Q15 a slice touched and what default + you chose) in the journal, not just in your head. +- The `sessions/` dir is committed to the repo (it is engineering history, not scratch). Never put + secrets, tokens, kubeconfigs, or customer data in a journal. + +### 3.4 `JOURNAL-TEMPLATE.md` (create verbatim) + +```markdown +# Session — + +**Builder:** · **Model/effort:** · **Branch:** feat/ +**Slice(s):** · **Status:** in-progress | done | blocked + +--- + +[G] Goal: +[S] Scope: +[A] Action: +[T] Test: +[C] Checkpoint #1: ; next: + + + +--- + +**Session close:** · **Open questions touched:** +``` + +--- + +## 4. Go style + +### 4.1 Formatting & imports + +- All Go is `gofmt`-clean and `goimports`-ordered. In golangci-lint v2 these run as **formatters** + (`golangci-lint fmt`), not linters. CI enforces "no diff" (§5). +- Local imports (`github.com/ArdurAI/sith/...`) group last; `goimports` local-prefix is set to the + module path. + +### 4.2 Linting — `.golangci.yml` (golangci-lint **v2**, pinned) + +Pin golangci-lint to a specific **v2.x** release in CI (§5). The config uses the v2 schema: + +```yaml +version: "2" + +run: + timeout: 5m + tests: true + +linters: + default: none + enable: + - errcheck # every returned error is handled or explicitly ignored + - govet # correctness vetting + - ineffassign # no ineffectual assignments + - staticcheck # the broad correctness/simplification suite (includes gosimple, stylecheck) + - unused # no dead code + - misspell # spelling in comments/strings + - revive # style: exported-symbol docs, naming, etc. + - gosec # security issues (matches Sith's threat posture) + - bodyclose # HTTP/response bodies are closed + - unconvert # no redundant conversions + settings: + revive: + rules: + - name: exported # exported symbols must be documented + - name: package-comments + - name: error-strings + - name: context-as-argument + - name: unreachable-code + misspell: + locale: US + exclusions: + generated: lax + rules: + # Test files may use unchecked errors on helpers and are exempt from gosec noise. + - path: _test\.go + linters: [errcheck, gosec] + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/ArdurAI/sith +``` + +> **Do not disable a linter to make code pass.** If a rule fires, fix the code. Narrow, justified +> `//nolint: // ` on a single line is allowed only with a written reason; blanket +> file/dir exclusions require an ADR note. + +### 4.3 Conventions + +- **Package names:** short, lower-case, no underscores; the directory name is the package name. +- **`internal/`** holds everything not meant for external import; only `cmd/sith` and `internal/*` + exist in Phase L. No `pkg/` until something is genuinely a public API. +- **Errors:** wrap with `fmt.Errorf("...: %w", err)`; never discard an error silently. Sentinel + errors are `var ErrX = errors.New(...)`; classify with `errors.Is/As`. +- **Context:** `ctx context.Context` is the first parameter of any function that does I/O or can + block. No `context.Background()` below `main`/command entry. +- **Logging:** structured only, via `log/slog` (§ `internal/logging`). No `fmt.Println` for + diagnostics; `fmt` is for user-facing command output only. +- **SPDX header:** every Go file starts with `// SPDX-License-Identifier: Apache-2.0`. +- **Fail-safe:** unknown/unschema'd input is refused, never defaulted-open. This is a product + invariant (`SITH-NOTION.md` §6 guardrail 5), enforced in code, not just docs. + +--- + +## 5. CI merge gates + +CI runs on every PR into `dev` and on pushes to `dev`. **All jobs must be green to merge.** The +gate set (see [`specs/SLICE-0-foundation.md`](specs/SLICE-0-foundation.md) §CI for the exact +workflow): + +1. **gofmt/format check** — `golangci-lint fmt --diff` reports no changes (fails on any diff). +2. **`go vet ./...`** — clean. +3. **golangci-lint** — the pinned v2.x run with the config above; zero findings. +4. **build** — `go build ./...` succeeds; `cmd/sith` produces a runnable binary. +5. **test** — `go test -race -count=1 ./...` passes. + +Additional merge requirements (branch protection): +- Every commit is **DCO signed-off** (a DCO check verifies each commit has a matching + `Signed-off-by`). +- Every commit is **SSH-signed** and verifies. +- At least one approving review (owner review counts). +- Branch is up to date with `dev` (rebased) before merge. +- No `--no-verify`, no `--no-gpg-sign`, no squash-merge (§1). + +Toolchain is **pinned**: the Go version in CI matches `go.mod`'s `go` directive; the golangci-lint +version is pinned in the workflow. Bumps to either are their own `ci:`/`build:` commit, reviewed +like any change. + +--- + +## 6. Test requirements + +- **Every non-trivial package ships tests.** For Phase L, `internal/config`, `internal/logging`, + `internal/buildinfo`, `internal/fleet`, and `internal/cli` all have tests from Slice 0. +- **Table-driven** tests where inputs vary; one behavior per test case with a descriptive name. +- **Deterministic and hermetic:** unit tests do no network and no real cluster I/O. Anything that + needs a live cluster, kubeconfig fan-out, or the network is an **e2e** test behind a build tag + (`//go:build e2e`) and is **not** part of the default `go test ./...` gate. +- **`-race` clean.** All tests pass under `go test -race`. +- **No `t.Skip` to dodge a failing assertion.** Skips are only for genuinely + environment-unavailable e2e paths and must state why. +- **Coverage targets (aspirational gates, not hard-failing in Phase L):** core logic packages + (`internal/fleet`, `internal/config`) aim for ≥ 70% statement coverage. Report coverage in CI; + tighten into a hard gate once the surface stabilizes. +- **Behavioral, not incidental:** test observable behavior (exit codes, printed output, typed + results, error classification), not private implementation details. + +--- + +## 7. The invariants that never bend (inherited from `SITH-NOTION.md`) + +These hold in **every** slice from Slice 0 onward; a PR that violates one does not merge: + +1. **Local mode is loopback-only.** `sith ui` / `sith serve --mcp` bind `127.0.0.1`/`::1` only. + Any routable bind in local mode is a bug. +2. **No account, no telemetry in local mode.** No phone-home, no analytics, nothing to opt out of + (Q15 default: permanent hard no — §`BUILD-SEQUENCE.md`). A network-egress test guards this. +3. **Credentials never leave the machine.** Kubeconfigs and exec-plugin creds are read in place, + never copied or uploaded. Persisted secrets go to the OS keychain, fail-loud, never silent + plaintext. +4. **Closed action vocabulary; no `exec`/free-form `apply` as *governed* actions.** Local per-pod + convenience ops (F11.5) use the user's own identity and are explicitly *not* governed intents. +5. **Fail-safe, never fail-open.** Unknown verb, unschema'd args, unresolved target, stale view, or + missing approval → refuse. diff --git a/docs/EPICS.md b/docs/EPICS.md new file mode 100644 index 0000000..23e0a3b --- /dev/null +++ b/docs/EPICS.md @@ -0,0 +1,3208 @@ +# Sith — Implementation Epics + +**Status:** planning · **Date:** 2026-07-08 · **License:** Apache-2.0 + +This document turns the Sith plan into a build order. It is a design artifact, not code — +the owner reviews it before any product code is written. It is downstream of, and +consistent with, the charter, architecture, scope, roadmap, threat model, competitive +analysis, and ADRs 0001–0006. Where those documents and this one seem to disagree, those +documents win and this one is wrong. + +Everything here respects the sequencing discipline the roadmap fixed and never violates: +**read before write · PR before mutation · exec never · prod never auto.** + +--- + +## 1. Overview + +**The one-sentence job:** give an operator one governed place to see and safely act across +many Kubernetes clusters, while deep cluster access stays local to each cluster. + +**Local-first dual mode (the reshape).** The same single Go binary is the product in two faces: +a **day-0 local client** (`sith` CLI/TUI + `sith ui` local web "fleet IDE") that federates the +user's own kubeconfig contexts with no hub, OCM, account, or telemetry — the **adoption +wedge** — and a **day-N hub** (`sith hub`) that adds OCM-brokered reach, tenancy, and governance +— the **durable moat**. They share one **source-abstract** fleet model and one enforcement +pipeline. New epics **E11** (local client), **E12** (connector framework), **E13** (cost +overlay) carry the reshape; rationale in +[`research/USE-CASE-AND-SHAPE.md`](research/USE-CASE-AND-SHAPE.md). + +Sith is a governed, multi-tenant, cross-cluster operations *federation* control plane for +generic Kubernetes fleets. It is **built on Open Cluster Management (OCM)** — it does not +re-implement cross-cluster transport, because `cluster-proxy` and `managed-serviceaccount` +already ship that, hardened and maintained. Sith is the governance, federation, and AI/MCP +layer on top. + +**Non-goals** (unchanged from `SCOPE.md`; if Sith drifts into any of these it has lost its +reason to exist): + +- Not a developer portal / IDP / service catalog (Backstage, Port, Cortex, OpsLevel). +- Not a GitOps controller / desired-state reconciler — Sith *opens PRs*, it does not + reconcile (Argo CD, Flux). +- Not a multi-cluster scheduler / workload-placement engine (Karmada, OCM Placement). +- Not a telemetry lake / metrics-logs backend — Sith *reads* health, it does not store + series (Prometheus, Grafana, Loki, Datadog). +- Not a bespoke cross-cluster tunnel — that is commodity plumbing, adopted from OCM. + +**The three federations** are what Sith owns, all built on OCM-brokered connectivity: + +1. **Read federation** — a tenant-scoped, normalized fleet model with cross-cluster + correlation as a first-class query. +2. **Action federation** — the only writes are typed intents from a closed verb + vocabulary, signed, re-validated locally by each spoke, executed with the spoke's own + scoped identity. No shell, no free-form apply, no secret/RBAC mutation. +3. **Policy federation** — fan-out reasoning: environment gates, wave/canary ordering with + a gate per wave, partial-failure and auto-rollback, idempotency, and + federation-specific abstention. + +[Ardur](https://github.com/ArdurAI/ardur) is the policy decision point (PDP), the broker +of scoped execution identity, and the decision-ledger (*why-allowed*) that complements +Sith's audit-log (*what-happened*). Sith is also exposed as a governed MCP server, so +external agents (Claude Code, Codex, kagent) become clients that inherit the same +governance. + +**Status of this doc.** All eleven epics below are planned; none are built. The headline +gate still stands: nothing in E1 onward is funded until E0 — the OCM falsification test — +returns yes. + +### Epic index + +| Epic | Name | Phase | Depends on | +|---|---|---|---| +| E0 | OCM substrate & falsification | M0 | — | +| E1 | Tenancy & identity | P1 | E0 | +| E2 | Read federation | P1 | E0, E1 | +| E3 | Credential & key custody | P1 → P2 | E1 | +| E4 | Action federation | P2 → P3 | E1, E2, E3, E5 | +| E5 | Policy federation & governance | P1 seam → P2 → P3 | E1, E2 (co-develops with E4) | +| E6 | Audit & decision ledger | P1 seam → P2 | E1, E5 | +| E7 | Governed MCP server | Phase L (read) → P2/P3 (write) | E2, E4, E5, E6 | +| E8 | Operator console / UI | Phase L → P3 | E2, E5, E6 | +| E9 | Deployment & packaging | Phase L → P3 | E0 | +| E10 | Observability & SRE for Sith itself | P1 → P3 | E9 | +| E11 | Local fleet client (adoption wedge) | **Phase L (day 0)** | E2 (shares the fleet model) | +| E12 | Connector framework | fast-follow (P2 → P3) | E2, E4 | +| E13 | Cost read-overlay | fast-follow (P3) | E2 | + +--- + +## 2. How to read this + +### The shape of each epic + +Every epic has an ID and name, a one-line goal, its roadmap phase, its dependencies, and a +list of features. Features are numbered `F.` (for example `F4.3` is the third +feature of epic E4). Each feature is written the same way, with these labelled parts: + +- **What it is** — one or two plain sentences. +- **How it works** — concrete, numbered steps describing the runtime behaviour, followed by + a Mermaid diagram: a `sequenceDiagram` when the point is who-calls-whom over time, a + `flowchart` when the point is branching logic and gates. +- **Acceptance criteria** — how we know the feature is done and correct. +- **Key risk / guardrail** — the one thing most likely to go wrong, and the control that + stops it. + +Each epic closes with epic-level exit criteria — the bar the whole epic clears before the +phase it belongs to is considered met. + +### Flowchart conventions + +- The hub components appear as `PEP`, `RF` (read federation), `AF` (action federation), + `FM` (fleet model), `MCP`, `AUD` (audit-log). Ardur appears as `PDP`, `IDB` (identity + broker), `LEDG` (decision-ledger). A spoke appears as `SP` (Sith spoke agent) and `SVC` + (cluster-local service such as Argo CD / Rollouts / Grafana). +- A decision node (`{ ... }`) with a `refuse` / `deny` / `abstain` branch is drawn on + almost every write path on purpose. Fail-safe is the default: the un-drawn "happy path" + is never the only path. +- Arrows *from* a spoke are always outbound (the spoke dials the hub). No diagram ever + shows the hub opening an inbound connection into a spoke — that property is load-bearing + and is preserved visually. + +### The closed action-verb vocabulary + +Writes are the only dangerous surface, so the set of things Sith can *do* is small, closed, +and reviewed. Every write in this document is one of these verbs and nothing else: + +| Verb | What it does | Idempotent | First shipped | +|---|---|---|---| +| `gitops.open-pr` | Opens a pull request against a target repo — a proposal a human merges | yes (dedupe by content) | P2 (first write) | +| `argocd.sync` | Triggers an Argo CD application sync to already-committed desired state | yes | P3 | +| `argocd.rollback` | Rolls an Argo CD application back to a previous synced revision | no | P3 | +| `rollout.promote` | Promotes an Argo Rollouts canary/blue-green to the next step | no | P3 | +| `rollout.abort` | Aborts an in-progress rollout and returns to stable | yes | P3 | +| `deployment.scale` | Sets replica count on a Deployment via the scale subresource | yes | P3 | +| `deployment.restart` | Triggers a rolling restart of a Deployment | no | P3 | + +**Permanently excluded, at every phase, by every actor including AI:** `exec` / shell into +a pod or node; free-form `kubectl apply` of arbitrary manifests; Secret +create / mutate / read-through; RBAC object mutation. These are not "not yet" — they are +not expressible in the model. Adding *any* new verb is an ADR-level decision +(`ADR-0004`); it is never a routine change. + +### The guardrails (the anti-drift contract, restated so every epic inherits it) + +1. If a capability ships as a maintained OCM addon or upstream project, adopt it rather + than build it. +2. If a feature belongs to "portal", "GitOps controller", "scheduler", or "telemetry + lake", it is out of scope — full stop. +3. The write path may only grow typed verbs in the reviewed closed vocabulary. `exec` and + free-form `apply` are permanently excluded. +4. Multi-tenant isolation, signed intents, per-spoke local enforcement, and scoped + identity are day-one requirements, not later hardening. +5. Fail-safe, never fail-open: anything not explicitly permitted is refused. Unknown verb, + unschema'd args, unresolved target, stale fleet view, or missing approval → refuse. +6. `prod` never auto-acts. Abstention ("I won't act, and here's why") is a first-class, + logged outcome, not an error. +7. The AI is a client of the governance, never a bypass of it. Its identity ceiling is + strictly below the human's; it never holds a cluster credential; it never gets a shell. + +## 3. Epics + +--- + +## E0 — OCM substrate and falsification + +**Goal:** prove, in a lab, that OCM's `cluster-proxy` + `managed-serviceaccount` deliver +outbound-only, cross-network, reach-cluster-local-services connectivity with scoped tokens — +so the whole "build a transport/agent" scope can be deleted. + +**Phase:** M0 · **Depends on:** nothing · **Nature:** a spike, not product code. The only +artifacts are a documented yes/no verdict and a reproducible runbook. No Sith product code is +written until this passes. + +**Features:** F0.1 hub + spoke lab provisioning · F0.2 OCM addon enablement · F0.3 reach a +spoke-local service through the tunnel · F0.4 scoped token projection · F0.5 outbound-only +verification · F0.6 falsification verdict and runbook. + +This epic exists to be able to fail cheaply. If any step cannot be made to work in about a day, +ADR-0001 moves to Rejected and the premise is re-examined before a line of product code exists. + +### F0.1 — Hub + spoke lab provisioning + +**What it is.** A local OCM environment — one hub cluster and two spokes (`spoke-a`, +`spoke-b`) on `kind` or `k3d` — with both spokes registered and healthy on the hub. + +**How it works.** +1. Create three local clusters. Keep all scratch state on `/Volumes/EXTENDED` (the system disk + is small). +2. Run `clusteradm init` on the hub to install the OCM hub control plane. +3. On each spoke, run the join command the hub emits; this installs the klusterlet and starts + an outbound registration request. +4. Accept each spoke's CSR on the hub (`clusteradm accept`). +5. Confirm two `ManagedCluster` objects report `Available`. +6. Capture every command and version into a gitignored runbook. + +```mermaid +flowchart TD + A["Create hub, spoke-a, spoke-b (kind/k3d on /Volumes/EXTENDED)"] --> B["clusteradm init on hub"] + B --> C["Run join command on each spoke (installs klusterlet)"] + C --> D["Spoke sends outbound registration / CSR to hub"] + D --> E["clusteradm accept on hub"] + E --> F{"Both ManagedCluster objects Available?"} + F -- "yes" --> G["Record commands + versions in runbook"] + F -- "no" --> H["Debug registration — if unworkable, flag ADR-0001 risk"] +``` + +**Acceptance criteria.** +- Two spokes are registered and show `Available` on the hub. +- A gitignored runbook reproduces the setup from scratch, with pinned versions. + +**Key risk / guardrail.** Registration friction (CSR, networking) can eat the time budget. +Guardrail: this is exactly the cheap place to hit it — time-box it and treat difficulty here as +a signal about OCM's operational cost, recorded in the runbook. + +### F0.2 — OCM addon enablement (`cluster-proxy` + `managed-serviceaccount`, pinned v0.10.0) + +**What it is.** The two OCM addons Sith depends on, enabled on the hub and both spokes, at +pinned versions verified for July 2026: `cluster-proxy` v0.10.0 and `managed-serviceaccount` +v0.10.0. + +**How it works.** +1. Enable the `cluster-proxy` addon; the hub runs proxy servers, each spoke runs a proxy agent + that dials out to the hub. +2. Enable the `managed-serviceaccount` addon on both spokes. +3. Wait for both addons to report `Available` on each spoke via their `ManagedClusterAddOn` + status. +4. Pin the versions in the runbook so the experiment and any later environment match. + +```mermaid +flowchart TD + A["Enable cluster-proxy addon (pin v0.10.0)"] --> B["Hub: proxy servers start"] + A --> C["Spokes: proxy agents start and dial hub"] + D["Enable managed-serviceaccount addon (pin v0.10.0)"] --> E["Spokes: MSA controller starts"] + B --> F{"Addon status Available on both spokes?"} + C --> F + E --> F + F -- "yes" --> G["Versions pinned in runbook"] + F -- "no" --> H["Inspect addon logs — do not proceed until healthy"] +``` + +**Acceptance criteria.** +- Both addons report `Available` on `spoke-a` and `spoke-b`. +- Versions are pinned and recorded. + +**Key risk / guardrail.** A version drift or a pre-release addon could behave differently from +the plan's assumptions. Guardrail: versions are pinned and any bump is an ADR-gated decision +(ADR-0001 update policy), not a silent upgrade. + +### F0.3 — Reach a spoke-local service through the `cluster-proxy` tunnel + +**What it is.** The deciding step: from the hub, reach an in-cluster service on a spoke (Grafana +or Argo CD, or a trivial stand-in) through the `cluster-proxy` reverse tunnel. + +**How it works.** +1. Deploy a small in-cluster service on each spoke (a plain HTTP service is enough; Grafana or + Argo CD makes the demo concrete). +2. From the hub, issue a request addressed through the `cluster-proxy` proxy service to the + spoke-local service. +3. The request travels the tunnel the spoke agent dialed; the response returns the same way. +4. Repeat for the second spoke to show it generalizes. +5. Time the whole end-to-end setup from F0.1 to here. + +```mermaid +sequenceDiagram + autonumber + participant HUB as Hub (curl / client) + participant CPS as cluster-proxy servers (hub) + participant AG as cluster-proxy agent (spoke, dialed out) + participant SVC as Spoke-local service (Grafana / Argo CD) + Note over AG,CPS: tunnel was DIALED spoke -> hub earlier (outbound-only) + HUB->>CPS: request addressed to spoke-local service + CPS->>AG: forward over the established reverse tunnel + AG->>SVC: reach service inside the spoke network + SVC-->>AG: response + AG-->>CPS: response over tunnel + CPS-->>HUB: response + Note over HUB: repeat for spoke-b, record total setup time +``` + +**Acceptance criteria.** +- The hub gets a valid response from an in-cluster service on both spokes, over the tunnel. +- The total setup time is measured and recorded. + +**Key risk / guardrail.** If reach requires opening an inbound path to the spoke, the core +premise fails. Guardrail: the reach must use only the tunnel the spoke dialed; any need for +inbound access is a falsification failure, recorded as such. + +### F0.4 — Scoped token projection (`managed-serviceaccount`) + +**What it is.** Authenticating that spoke-local reach with a short-lived, scoped +`managed-serviceaccount` token projected to the hub — never a cluster-admin kubeconfig. + +**How it works.** +1. Create a `ManagedServiceAccount` on each spoke scoped to only what the read demo needs. +2. The MSA addon provisions the ServiceAccount on the spoke and projects its token back to the + hub as a secret, with a chosen audience. +3. The hub uses that projected token to authenticate the F0.3 request to the spoke-local + service. +4. Confirm the token is scoped (not cluster-admin) and rotates. + +```mermaid +sequenceDiagram + autonumber + participant HUB as Hub + participant MSA as managed-serviceaccount addon + participant SP as Spoke + participant SVC as Spoke-local service + HUB->>MSA: request a scoped ManagedServiceAccount on the spoke + MSA->>SP: create scoped ServiceAccount + mint token + SP-->>MSA: token (short-lived, scoped, chosen audience) + MSA-->>HUB: project token to hub as a secret + HUB->>SVC: authenticated reach using the projected scoped token + SVC-->>HUB: response + Note over HUB,SP: no cluster-admin kubeconfig ever leaves the spoke +``` + +**Acceptance criteria.** +- Reach in F0.3 is authenticated with a projected, scoped MSA token. +- No cluster-admin kubeconfig is used anywhere in the flow; the token is scoped and rotatable. + +**Key risk / guardrail.** A too-broad token would quietly reintroduce the god-credential +anti-pattern. Guardrail: the MSA is scoped to the minimum the demo needs, and "no admin +kubeconfig in the center" is verified as an explicit check, not assumed. + +### F0.5 — Outbound-only verification + +**What it is.** Evidence that the spoke only ever makes outbound connections to the hub — the +property that lets spokes live in isolated VPCs or behind NAT. + +**How it works.** +1. With the tunnel established and reach working, inspect the spoke's live connections with + `ss` / `netstat`, and optionally `tcpdump`. +2. Confirm every hub-directed connection is outbound (dialed by the spoke). +3. Confirm no inbound hub → spoke port is required for reach to work. +4. Record the observation as evidence for ADR-0001. + +```mermaid +flowchart TD + A["Tunnel up, reach working"] --> B["Inspect spoke connections (ss / netstat / tcpdump)"] + B --> C{"All hub-directed connections outbound?"} + C -- "yes" --> D{"Any inbound hub -> spoke port required?"} + C -- "no" --> X["Falsification concern: not outbound-only — record and escalate"] + D -- "no" --> E["Record outbound-only evidence in ADR-0001"] + D -- "yes" --> X +``` + +**Acceptance criteria.** +- Documented evidence that spoke → hub traffic is outbound-only and no inbound port is needed. + +**Key risk / guardrail.** A hidden inbound dependency would undermine the isolated-VPC claim. +Guardrail: the check is explicit and adversarial (look for inbound requirements, do not just +confirm the happy path), and a negative result is a real finding. + +### F0.6 — Falsification verdict and runbook + +**What it is.** Turning the experiment into a durable decision: fill ADR-0001's falsification +section, move its status, and commit a reproducible runbook plus a short demo capture. + +**How it works.** +1. Record the result, the setup time, and notes in ADR-0001's "Falsification evidence" section. +2. If reachable within about a day, move ADR-0001 to Accepted and delete the transport-build + scope; proceed to Phase 1. If not, move it to Rejected and stop for re-evaluation. +3. Commit the redacted runbook (no secrets) so the experiment is reproducible. +4. Record a short terminal capture: hub reaching a spoke-local service through `cluster-proxy` + with an MSA token, and the outbound-only evidence. + +```mermaid +flowchart TD + A["Experiment complete (F0.1–F0.5)"] --> B{"Reachable via cluster-proxy + MSA in ~1 day?"} + B -- "yes" --> C["ADR-0001 -> Accepted"] + C --> D["Delete 'build the transport' scope"] + D --> E["Proceed to Phase 1 (E1, E2)"] + B -- "no" --> F["ADR-0001 -> Rejected"] + F --> G["Stop — re-evaluate premise before any product code"] + C --> H["Commit runbook + demo capture as evidence"] + F --> H +``` + +**Acceptance criteria.** +- ADR-0001 carries a real verdict (Accepted or Rejected), a setup time, and a link to the + runbook. +- The go/no-go decision for Phase 1 is explicit and recorded. + +**Key risk / guardrail.** The temptation after a "no" is to build the transport anyway. +Guardrail: a "no" verdict is a hard stop for re-evaluation, by design — the whole point of M0 is +that this is the cheapest place to abandon or pivot the premise. + +### E0 exit criteria + +- OCM hub + 2 spokes stand up, both `Available`; both addons healthy at pinned v0.10.0. +- The hub reaches a spoke-local service through the `cluster-proxy` tunnel using a scoped MSA + token, on both spokes, with no cluster-admin kubeconfig anywhere. +- Spoke → hub traffic is verified outbound-only with no inbound port required. +- ADR-0001 records the verdict and setup time; a redacted runbook and a demo capture exist. +- The transport-build scope is deleted (on "yes"), or work stops for re-evaluation (on "no"). + +## E1 — Tenancy and identity + +**Goal:** make "a workspace over many clusters" the single tenancy anchor, with authorization +from signed token claims (never headers), least-privilege RBAC roles, and a real database-level +row-level-security backstop behind app-layer scoping. + +**Phase:** P1 · **Depends on:** E0 (accepted). Isolation is the product here; a control plane +that can see and act across many tenants' fleets must never leak or act across the tenant +boundary. This epic implements the three-layer defense of ADR-0003 from day one. + +**Features:** F1.1 workspace + membership model · F1.2 signed-token authentication · F1.3 RBAC +role gate · F1.4 application-layer tenant scoping · F1.5 database-level RLS backstop · F1.6 +tenant-isolation test suite. + +### F1.1 — Workspace + membership model + +**What it is.** The tenancy data model. A `Workspace` is the scoped tenancy object; every +cluster, policy, intent, decision, audit entry, and fleet fact belongs to exactly one workspace. +A `Membership` grants a subject a role within a workspace. + +**How it works.** +1. `Workspace` carries a `tenant_key` that is the isolation anchor used by every scoping layer. +2. Clusters are explicitly associated with a workspace (tenancy is a workspace over clusters, + never one deployment per cluster). +3. `Membership` maps a subject to a role in a workspace: `reader`, `operator`, `approver`, or + `admin`. +4. Every workspace-scoped table carries a `workspace_id` foreign key so all three isolation + layers have a column to enforce on. + +```mermaid +erDiagram + WORKSPACE ||--o{ CLUSTER : "scopes" + WORKSPACE ||--o{ MEMBERSHIP : "grants" + WORKSPACE ||--o{ POLICY : "owns" + WORKSPACE ||--o{ INTENT : "issues" + WORKSPACE { + id id PK + string name + string tenant_key "isolation anchor" + } + MEMBERSHIP { + id id PK + id workspace_id FK + string subject + string role "reader|operator|approver|admin" + } + CLUSTER { + id id PK + id workspace_id FK + string ocm_managedcluster_ref + } +``` + +**Acceptance criteria.** +- Every workspace-scoped entity carries a `workspace_id`; a cluster belongs to exactly one + workspace. +- A subject's role in a workspace is expressed only through `Membership`. + +**Key risk / guardrail.** A new workspace-scoped table added later without a `workspace_id` +would be a silent leak path. Guardrail: a CI guard (F1.4) forbids workspace-scoped tables that +are not wired into scoping, so an omission fails the build. + +### F1.2 — Signed-token authentication (no header trust) + +**What it is.** Identity, tenant, and role come only from a cryptographically verified token. +Request headers are never trusted for identity — a direct fix for the predecessor's +header-trust IDOR. + +**How it works.** +1. The gateway verifies the session/token signature before anything else; an invalid or absent + token is rejected. +2. Tenant and role are read from the token's `memberships[workspace] → role` claim. +3. Any inbound `x-*-role` / `x-*-tenant` headers are stripped or ignored; they have no effect on + authorization. +4. The verified claims flow downstream as the only source of who-and-where. + +```mermaid +sequenceDiagram + autonumber + participant C as Client (UI / MCP / CLI) + participant GW as API gateway + participant H as Downstream handlers + C->>GW: request (+ signed token, maybe spoofed x-*-role header) + GW->>GW: verify token signature + alt invalid / absent token + GW-->>C: 401 reject + else valid + GW->>GW: read tenant + role from token claims + GW->>GW: strip / ignore any x-*-role, x-*-tenant headers + GW->>H: pass verified claims (headers have no effect) + end +``` + +**Acceptance criteria.** +- A header-injected role has no effect; identity and tenant come only from the signed token. +- A forged or absent token is rejected. + +**Key risk / guardrail.** A single handler reading a header for authorization would reopen the +IDOR. Guardrail: headers are stripped at the gateway and the isolation test suite (F1.6) asserts +a header-injected role changes nothing. + +### F1.3 — RBAC role gate (least privilege) + +**What it is.** A gate that maps the actor's workspace role to the classes of action they may +take, applied before any read or write proceeds. + +**How it works.** +1. `reader` may run reads and correlation queries only. +2. `operator` may additionally propose intents. +3. `approver` may approve gated intents (and read), but the proposer and approver must be + distinct for multi-approver gates. +4. `admin` manages workspace membership and policy bindings, within the workspace only. +5. The gate is fail-safe: a role that does not explicitly permit an action is refused. + +```mermaid +flowchart TD + A["Verified actor + role (from F1.2)"] --> B{"Role permits this action class?"} + B -- "reader: reads only" --> R{"Is this a read?"} + B -- "operator: reads + propose" --> P{"Read or propose-intent?"} + B -- "approver: reads + approve" --> V{"Read or approve?"} + B -- "admin: + manage members/policy (in-workspace)" --> M["Allow scoped admin action"] + R -- "yes" --> OK["Proceed"] + R -- "no" --> DENY["Refuse (fail-safe)"] + P -- "yes" --> OK + P -- "no" --> DENY + V -- "yes" --> OK + V -- "no" --> DENY +``` + +**Acceptance criteria.** +- Each role can do exactly its permitted action classes and no more. +- An action a role does not permit is refused, not best-effort. + +**Key risk / guardrail.** Role creep (an operator quietly gaining approval power) collapses +separation of duties. Guardrail: proposer and approver identities are checked distinct for +multi-approver gates, and the role gate is fail-safe. + +### F1.4 — Application-layer tenant scoping + +**What it is.** A tenant-aware data access layer that injects the current workspace scope into +every query against a workspace-scoped table and hard-fails on any mismatch — covering all such +models, not a subset. + +**How it works.** +1. Every request runs with a resolved workspace context (from F1.2). +2. The data access layer injects `workspace_id = ` into reads and writes on + workspace-scoped tables. +3. If a query returns or targets a row from another workspace, the layer hard-fails rather than + returning it. +4. A CI guard scans for direct access to workspace-scoped tables that bypasses the scoped layer + and fails the build. + +```mermaid +flowchart TD + A["Query on a workspace-scoped table"] --> B["Tenant-aware DAL injects workspace_id = current"] + B --> C{"Any row outside current workspace?"} + C -- "no" --> D["Return / apply within workspace"] + C -- "yes" --> E["Hard-fail (do not return foreign rows)"] + F["CI guard: scan for un-scoped table access"] --> G{"Direct/un-scoped access found?"} + G -- "yes" --> H["Fail the build"] + G -- "no" --> I["Build passes"] +``` + +**Acceptance criteria.** +- All workspace-scoped models are accessed only through the scoped layer. +- A cross-workspace access attempt hard-fails at the app layer; the CI guard blocks un-scoped + access. + +**Key risk / guardrail.** A forgotten filter on one model was the predecessor's silent-leak bug. +Guardrail: the CI guard makes an un-scoped access a build failure, and F1.5 is the independent +backstop if the app layer is ever wrong anyway. + +### F1.5 — Database-level RLS backstop (non-owner role, FORCE RLS, per-request scope) + +**What it is.** PostgreSQL row-level security that is actually enforced, independent of +application code — the backstop the predecessor advertised but left inert. + +**How it works.** +1. The application connects as a non-owner DB role (table owners bypass RLS, so the app must not + be the owner). +2. Every workspace-scoped table has `ENABLE` and `FORCE ROW LEVEL SECURITY`. +3. At the start of each request's transaction, the current workspace is set with + `set_config('sith.workspace_id', , true)`. +4. Each table's RLS policy checks `workspace_id = current_setting('sith.workspace_id')`, so the + database filters foreign rows even if the app layer is bypassed. + +```mermaid +sequenceDiagram + autonumber + participant APP as App (non-owner DB role) + participant DB as PostgreSQL (FORCE RLS) + APP->>DB: BEGIN transaction + APP->>DB: set_config('sith.workspace_id', W, true) + APP->>DB: SELECT / INSERT on workspace-scoped table + DB->>DB: RLS policy: workspace_id = current_setting('sith.workspace_id') + DB-->>APP: only rows for workspace W (foreign rows filtered by DB) + APP->>DB: COMMIT + Note over APP,DB: owner role would bypass RLS — app deliberately is NOT the owner +``` + +**Acceptance criteria.** +- The app connects as a non-owner role; every workspace-scoped table has `FORCE ROW LEVEL + SECURITY`. +- A query with the app-layer scope deliberately removed still returns only the current + workspace's rows. + +**Key risk / guardrail.** RLS that is enabled but not forced, or an app connecting as owner, +would make the backstop inert again. Guardrail: F1.6 includes a test that removing an RLS policy +makes a DB-layer isolation test fail — proving the backstop is live, not decorative. + +### F1.6 — Tenant-isolation test suite + +**What it is.** The test suite that treats isolation as a primary, first-class property, proving +cross-workspace access is impossible at multiple independent layers. + +**How it works.** +1. A cross-workspace read/write is attempted with the app-layer scope deliberately bypassed; the + DB RLS layer must deny it. +2. A forged or absent token is rejected; a header-injected role is shown to have no effect. +3. `targetSelector` and queries are fuzzed with foreign cluster IDs and must always resolve to + empty within-workspace. +4. A negative control: a removed RLS policy makes the DB-layer test fail, proving the backstop is + real. + +```mermaid +flowchart TD + A["Isolation test suite"] --> B["Case 1: app-bypassed cross-workspace query -> DB RLS denies"] + A --> C["Case 2: forged/absent token rejected, header role has no effect"] + A --> D["Case 3: fuzz targetSelector with foreign cluster IDs -> empty"] + A --> E["Negative control: remove an RLS policy"] + B --> F{"All green?"} + C --> F + D --> F + E --> G{"DB-layer test now FAILS?"} + F -- "yes" --> PASS["Isolation verified"] + G -- "yes" --> PASS + G -- "no" --> FAIL["RLS is not actually enforced — fix before shipping"] +``` + +**Acceptance criteria.** +- All isolation cases pass; a removed RLS policy makes the DB-layer test fail. +- Header-injected roles and forged tokens are proven ineffective. + +**Key risk / guardrail.** A green suite that does not actually exercise the DB layer would give +false confidence. Guardrail: the negative control (remove-a-policy-and-watch-it-fail) is part of +the suite, so "green" means the backstop is genuinely enforcing. + +### E1 exit criteria + +- `Workspace` is the single tenancy anchor; every scoped entity carries `workspace_id`. +- Authorization derives only from signed token claims; header-injected roles have no effect. +- RBAC roles (reader/operator/approver/admin) gate action classes fail-safe. +- App-layer scoping covers all workspace-scoped models with a CI guard; DB-level RLS is forced, + per-request, and connects as a non-owner role. +- The isolation suite is green, including the negative control proving RLS is live. + +## E2 — Read federation + +**Goal:** assemble a tenant-scoped, normalized fleet model from OCM-brokered reads and make +cross-cluster correlation a first-class query — answering fleet-wide questions single-cluster +tools structurally cannot. The read source is **abstracted**: facts come from a **local +kubeconfig context** (day-0 local mode, E11) *or* an **OCM-brokered spoke** (day-N hub mode), +behind one common source interface — so the local client and the hub are one code path above +the source. F2.1 defines both source adapters. + +**Phase:** P1 · **Depends on:** E0 (connectivity), E1 (tenancy). Reads never require the write +path and have their own blast radius and rate limits. Least privilege is by construction: the +hub reads only what spoke agents report and only within the scope of the projected MSA token. + +**Features:** F2.1 spoke read collection and normalization · F2.2 fleet model store with +freshness and source stamping · F2.3 cross-cluster correlation query · F2.4 image/CVE fact +ingestion and fleet-wide CVE search · F2.5 staleness surfacing and abstention inputs. + +### F2.1 — Spoke read collection and normalization + +**What it is.** The read path that pulls inventory (deployments, pods, rollouts) and +health/alerts from each spoke through `cluster-proxy` using a scoped MSA token, and normalizes +it into a common shape. + +**How it works.** +1. For each spoke, the read-federation service authenticates with the spoke's projected MSA + token and reaches the spoke's Kubernetes API / cluster-local services through + `cluster-proxy`. +2. It collects inventory (workload objects and their status) and health/alert signals. +3. It maps heterogeneous source shapes into Sith's normalized fleet-fact model (kind = + inventory | health | alert | drift | cve). +4. Reads are bounded to what the MSA token scope and the spoke report allow — the hub cannot + read arbitrary cluster state just because it is the hub. + +```mermaid +sequenceDiagram + autonumber + participant RF as Read-federation service (hub) + participant MSA as MSA token store (hub) + participant CP as cluster-proxy + participant SVC as Spoke API / local services + RF->>MSA: get scoped token for spoke + MSA-->>RF: short-lived scoped token + RF->>CP: read inventory + health (authenticated, scoped) + CP->>SVC: reach over reverse tunnel + SVC-->>CP: raw inventory / health / alerts + CP-->>RF: results (bounded by token scope) + RF->>RF: normalize into fleet-fact model + Note over RF: repeat per spoke, scope limits what can be read +``` + +**Acceptance criteria.** +- Inventory and health are collected from ≥ 2 spokes over `cluster-proxy` with scoped tokens. +- Heterogeneous sources are normalized into one fleet-fact shape. + +**Key risk / guardrail.** A broad token would let the hub over-read a spoke. Guardrail: reads +are bounded by the MSA token scope and by what the spoke agent chooses to report; the hub holds +no admin path. + +### F2.2 — Fleet model store with freshness and source stamping + +**What it is.** The tenant-scoped, cached, normalized fleet model where every record carries the +time it was observed and the cluster it came from. + +**How it works.** +1. Each normalized fact is written to the fleet model with `observed_at` and `source cluster`. +2. Every fact is scoped to its cluster's workspace, so the model inherits E1's isolation + (app-layer scope + DB RLS). +3. The model is a cache of observed state, refreshed on a cadence — it is not a metrics store and + keeps no long time series. +4. `CLUSTER.last_seen` is maintained so a cluster that stops reporting is detectable. + +```mermaid +flowchart TD + A["Normalized facts (from F2.1)"] --> B["Stamp observed_at + source cluster"] + B --> C["Scope to workspace (app layer + DB RLS)"] + C --> D[("Fleet model cache: inventory/health/alert/drift/cve")] + D --> E["Update CLUSTER.last_seen"] + E --> F{"Cluster stopped reporting?"} + F -- "yes" --> G["last_seen ages -> staleness input (F2.5)"] + F -- "no" --> H["Fresh record available to queries"] +``` + +**Acceptance criteria.** +- The fleet model is populated from ≥ 2 spokes, tenant-scoped, each record stamped with freshness + and source. +- The store holds current observed state only — no long-term series. + +**Key risk / guardrail.** Drifting toward storing metrics history would turn Sith into a +telemetry lake (out of scope). Guardrail: the model is a bounded cache of current facts; +retention is deliberately short and history is not a feature. + +### F2.3 — Cross-cluster correlation query + +**What it is.** The differentiator: a query that spans all clusters in a workspace and answers a +question a single-cluster tool cannot, such as "every cluster where deployment `X` is unhealthy". + +**How it works.** +1. A query names a condition (for example, deployment `X` in a Degraded/unhealthy state). +2. The query engine evaluates it against the workspace's fleet model across every cluster at + once — no per-console fan-out by the operator. +3. Results are aggregated into one answer listing the matching clusters. +4. Any cluster whose data is stale is flagged in the result, so the answer is honest about + coverage. + +```mermaid +flowchart TD + A["Query: every cluster where X is unhealthy"] --> B["Resolve within actor's workspace only"] + B --> C["Evaluate condition across all clusters in fleet model"] + C --> D["Aggregate matches into one answer"] + D --> E{"Any matching/relevant cluster stale?"} + E -- "yes" --> F["Flag stale clusters in the result"] + E -- "no" --> G["Return complete cross-cluster answer"] + F --> G +``` + +**Acceptance criteria.** +- One query returns a correct, tenant-scoped answer spanning ≥ 2 spokes. +- Stale clusters are flagged in the result rather than silently dropped. + +**Key risk / guardrail.** A query that silently omits a stale or dark cluster would give a +false-complete answer. Guardrail: staleness is surfaced in every result (F2.5), and completeness +of coverage is explicit. + +### F2.4 — Image/CVE fact ingestion and fleet-wide CVE search + +**What it is.** Ingestion of image and CVE facts per cluster, enabling a fleet-wide search such +as "which clusters run image X with CVE Y". + +**How it works.** +1. Each spoke reports the images running in its cluster (and, where available, associated CVE + findings) as `cve`/`inventory` fleet facts. +2. The hub normalizes and stores these with the same freshness/source stamping. +3. A CVE-search query filters across the workspace's clusters by image or CVE identifier. +4. Results list the matching clusters and workloads, with staleness flagged. + +```mermaid +sequenceDiagram + autonumber + participant SP as Spoke report + participant RF as Read-federation service + participant FM as Fleet model + participant Q as CVE-search query + SP->>RF: image list (+ CVE findings where available) + RF->>FM: store as cve/inventory facts (freshness + source) + Q->>FM: search image X / CVE Y across workspace clusters + FM-->>Q: matching clusters + workloads (stale flagged) + Note over Q,FM: single-cluster tools cannot answer this fleet-wide +``` + +**Acceptance criteria.** +- A fleet-wide search by image or CVE returns the matching clusters across the workspace. +- Results carry freshness and source; stale clusters are flagged. + +**Key risk / guardrail.** Treating a stale image inventory as authoritative could hide a +vulnerable cluster. Guardrail: CVE results inherit freshness stamping, and a stale cluster is +flagged rather than assumed clean. + +### F2.5 — Staleness surfacing and abstention inputs + +**What it is.** The mechanism that turns per-record freshness into a per-cluster staleness signal +and feeds it to correlation results and, later, to action-side abstention. + +**How it works.** +1. Each cluster's freshness is derived from `last_seen` / `observed_at` versus a configured + threshold. +2. A cluster past the threshold is marked stale; the degree of staleness (for example, ">10m") + is available. +3. Query results carry per-cluster staleness so operators see coverage gaps. +4. The same signal is exported for the policy layer, where an incomplete or stale targeted set + drives abstention (E5). + +```mermaid +flowchart TD + A["Per-record observed_at / CLUSTER.last_seen"] --> B["Compare to freshness threshold"] + B --> C{"Past threshold?"} + C -- "yes" --> D["Mark cluster stale (with age, e.g. >10m)"] + C -- "no" --> E["Cluster fresh"] + D --> F["Surface staleness in read results"] + D --> G["Export as abstention input to policy layer (E5)"] + E --> F +``` + +**Acceptance criteria.** +- Per-cluster staleness is computed and visible in read results. +- The staleness signal is available to the policy layer as an abstention input. + +**Key risk / guardrail.** Hidden staleness is the most dangerous failure of a federated read +model. Guardrail: staleness is never hidden — it is surfaced in results and is a first-class +input to abstention, so a partial view cannot masquerade as a complete one. + +### E2 exit criteria + +- A normalized, tenant-scoped fleet model is populated from ≥ 2 spokes, every record stamped with + freshness and source. +- A single cross-cluster correlation query returns a correct answer over ≥ 2 spokes with stale + clusters flagged. +- Fleet-wide image/CVE search works across the workspace. +- Per-cluster staleness is surfaced in results and exported as an abstention input. +- Reads never touch the write path and are bounded by MSA token scope. + +## E3 — Credential and key custody + +**Goal:** hold as few secrets as possible, and protect the ones that remain with envelope +encryption and per-tenant keys so no single leak decrypts every tenant — and with no god +credential in the center. + +**Phase:** the *principles* are day-one (P1); the concrete custody surfaces land when the first +held secret arrives with the write path (P2 — Git credentials and the intent signing key). +**Depends on:** E1 (workspaces are the key-scoping unit). This epic makes the predecessor's +single-env-key blast radius structurally impossible (ADR-0006). + +**Features:** F3.1 no-central-admin-credential posture · F3.2 KMS envelope encryption with +per-tenant data keys · F3.3 intent signing-key custody · F3.4 key rotation and key-ring · F3.5 +boot-time custody checks · F3.6 secret leak prevention. + +### F3.1 — No-central-admin-credential posture + +**What it is.** The structural stance that the hub stores no per-cluster admin kubeconfigs at +all; reach uses scoped MSA tokens and action uses short-lived brokered identity, verified and +executed locally by the spoke. + +**How it works.** +1. Cluster reach = OCM `managed-serviceaccount` scoped tokens (E0/E2), not stored admin + kubeconfigs. +2. Cluster action = Ardur-brokered, short-lived, per-action identity (E5), re-validated and + executed by the spoke with its own local identity. +3. The hub therefore never holds a credential whose compromise means cluster-admin everywhere. +4. Any secret the hub genuinely must hold (Git credential, signing key, integration token) is a + named, bounded exception handled by F3.2–F3.3. + +```mermaid +flowchart TD + A["What does the hub hold?"] --> B["Cluster-admin kubeconfigs? NO — not stored"] + A --> C["Cluster reach -> scoped MSA token (short-lived)"] + A --> D["Cluster action -> Ardur-brokered per-action identity"] + A --> E["Only named exceptions held: Git cred, signing key, integration token"] + E --> F["Each exception protected by KMS envelope (F3.2) / KMS-HSM (F3.3)"] + B --> G["Center compromise != cluster-admin everywhere"] + C --> G + D --> G +``` + +**Acceptance criteria.** +- No per-cluster admin kubeconfig is stored in the hub. +- Every secret the hub does hold is an enumerated exception with a defined custody control. + +**Key risk / guardrail.** A convenience shortcut ("just store the kubeconfig") would reintroduce +the confused-deputy blast radius. Guardrail: storing cluster-admin credentials centrally is +rejected outright by ADR-0001/ADR-0006; reach and action use scoped, brokered, locally +re-validated identity only. + +### F3.2 — KMS envelope encryption with per-tenant data keys + +**What it is.** Any secret the hub holds is encrypted with a per-workspace data key, which is +itself wrapped by a KMS/HSM master key. There is no single process-wide key. + +**How it works.** +1. Each workspace has its own data key (DEK). To store a secret, the DEK encrypts the plaintext + (AES-256-GCM). +2. The DEK is never stored in the clear — it is wrapped (encrypted) by the KMS master key, which + never leaves the KMS. +3. Stored form is the ciphertext plus the wrapped DEK; to read, the KMS unwraps the DEK, which + then decrypts the secret. +4. Compromising one tenant's DEK exposes only that tenant; the KMS master key is never present + in process memory in the clear. + +```mermaid +sequenceDiagram + autonumber + participant APP as Hub + participant KMS as KMS / HSM (master key never leaves) + Note over APP,KMS: ENCRYPT a secret for workspace W + APP->>APP: DEK_W encrypts plaintext (AES-256-GCM) + APP->>KMS: wrap DEK_W with master key + KMS-->>APP: wrapped DEK_W + APP->>APP: store {ciphertext, wrapped DEK_W} + Note over APP,KMS: DECRYPT later + APP->>KMS: unwrap DEK_W + KMS-->>APP: DEK_W (in-process, transient) + APP->>APP: DEK_W decrypts ciphertext +``` + +**Acceptance criteria.** +- Every hub-held secret is encrypted with a per-workspace DEK wrapped by a KMS master key. +- No single process-wide key exists; one tenant's DEK compromise does not expose others. + +**Key risk / guardrail.** A per-tenant key kept in the DB unwrapped would recreate a single point +of compromise. Guardrail: DEKs are always KMS-wrapped at rest; the master key never leaves the +KMS, so the key store alone is not sufficient to decrypt anything. + +### F3.3 — Intent signing-key custody + +**What it is.** The key the hub uses to sign every dispatched intent (E4), held in a KMS/HSM, +treated as the single most sensitive secret in the system. + +**How it works.** +1. The signing key lives in a KMS/HSM; signing is a KMS operation, so the private key never + enters application memory in the clear. +2. Each intent is signed by calling the KMS to produce a signature; each spoke verifies it. +3. The key is rotatable (F3.4); spoke-side local allowlists are the compensating control if the + key is ever compromised. +4. Access to the signing operation is tightly scoped and audited. + +```mermaid +flowchart TD + A["Hub prepares intent to dispatch"] --> B["Request signature from KMS/HSM"] + B --> C["KMS signs (private key never leaves KMS)"] + C --> D["Signed intent dispatched to spoke (E4)"] + D --> E["Spoke verifies signature before acting"] + F["If signing key compromised"] --> G["Spoke local allowlist bounds damage to already-permitted verbs/targets"] +``` + +**Acceptance criteria.** +- Intents are signed via a KMS/HSM operation; the private key never leaves the KMS. +- The signing key is rotatable and its use is audited. + +**Key risk / guardrail.** The signing key is the highest-value secret — its leak lets an attacker +forge intents. Guardrail: it lives in KMS/HSM, rotates, and the spoke's independent local +allowlist (E4) bounds the damage of any forged intent to already-permitted verbs and targets. + +### F3.4 — Key rotation and key-ring + +**What it is.** First-class rotation for data keys and the signing key, with a key-ring that can +decrypt with an old key while encrypting with a new one during rotation. + +**How it works.** +1. Keys carry versions; a key-ring holds the current and recent versions. +2. On a schedule or on demand, a new key version is created and becomes the encrypt/sign key. +3. Existing ciphertext/signatures are still verifiable/decryptable with the older version until + re-wrapped/re-signed. +4. Rotation is a primary test target, since a rotation bug can lock out data or break signature + verification. + +```mermaid +flowchart TD + A["Rotation trigger (schedule or on-demand)"] --> B["Create new key version v(n+1)"] + B --> C["Encrypt / sign new data with v(n+1)"] + B --> D["Keep v(n) in key-ring for decrypt/verify of old data"] + C --> E["Background re-wrap / re-sign old data to v(n+1)"] + D --> E + E --> F{"All migrated to v(n+1)?"} + F -- "yes" --> G["Retire v(n)"] + F -- "no" --> D +``` + +**Acceptance criteria.** +- Data keys and the signing key rotate on schedule and on demand without data loss or broken + verification. +- Old versions remain usable for decrypt/verify until migration completes. + +**Key risk / guardrail.** A rotation that drops an old key before re-encryption would strand data +or break verification. Guardrail: the key-ring keeps old versions until migration completes, and +rotation is exercised explicitly in tests. + +### F3.5 — Boot-time custody checks + +**What it is.** Startup checks that refuse to run if key material is missing, weak, a +placeholder, or if the KMS is unreachable. + +**How it works.** +1. On boot, the hub verifies required key material is present and above an entropy floor. +2. It rejects any placeholder or well-known default value (no "changeme" ever accepted). +3. It verifies KMS reachability so envelope operations will work. +4. If any check fails, the process refuses to start rather than running in a degraded, unsafe + state. + +```mermaid +flowchart TD + A["Process start"] --> B{"Key material present?"} + B -- "no" --> X["Refuse to start"] + B -- "yes" --> C{"Above entropy floor and not a placeholder?"} + C -- "no" --> X + C -- "yes" --> D{"KMS reachable?"} + D -- "no" --> X + D -- "yes" --> E["Start normally"] +``` + +**Acceptance criteria.** +- The hub refuses to start on missing, weak, or placeholder key material, or an unreachable KMS. +- A known-weak or placeholder value is rejected. + +**Key risk / guardrail.** Silently starting with a weak key is how the predecessor accepted bad +key material. Guardrail: boot checks are fail-closed — an unsafe custody state prevents startup +rather than degrading quietly. + +### F3.6 — Secret leak prevention + +**What it is.** The controls that keep secrets out of logs, errors, git, and Helm output — this +is a public repository, so nothing sensitive is ever committed. + +**How it works.** +1. An error sanitizer strips tokens, keys, and sensitive identifiers before anything is logged or + returned. +2. Secrets are never rendered into log lines or into Helm output that lands in git. +3. `.gitignore` pre-empts common secret files; scratch/lab state stays out of the repo. +4. This applies uniformly across control plane, spoke agent, and tooling. + +```mermaid +flowchart TD + A["Log line / error / rendered output"] --> B["Error sanitizer: strip tokens/keys/sensitive IDs"] + B --> C{"Any secret material remaining?"} + C -- "yes" --> D["Redact before emit"] + C -- "no" --> E["Emit safe output"] + F["Repo hygiene: .gitignore + no secrets in Helm output"] --> G["Public repo: nothing sensitive committed"] +``` + +**Acceptance criteria.** +- Logs, errors, and rendered output never contain secret material. +- No secret files are committed; the public repo stays clean. + +**Key risk / guardrail.** A single unsanitized error path can leak a token (the SSRF-reads-env +class of bug). Guardrail: sanitization is centralized and applied to all emit paths, and repo +hygiene is enforced so a leak cannot reach git. + +### E3 exit criteria + +- The hub stores no cluster-admin kubeconfigs; reach and action use scoped/brokered identity. +- Every hub-held secret is envelope-encrypted with a per-tenant DEK wrapped by a KMS master key. +- The intent signing key lives in KMS/HSM and is rotatable; rotation works via a key-ring with no + data loss. +- Boot-time custody checks fail closed; secrets never reach logs, errors, or git. + +## E4 — Action federation + +**Goal:** make the only writes Sith performs be typed intents from a closed verb vocabulary — +signed by the hub, re-validated independently by each spoke, executed with the spoke's own scoped +identity — with `gitops.open-pr` as the first and safest write and no shell ever. + +**Phase:** P2 (the `gitops.open-pr` path) → P3 (live-mutation verbs behind full policy). +**Depends on:** E1 (identity/tenancy), E2 (target resolution against the fleet model), E3 (Git +credential + signing key custody), and E5 (every intent rides the PEP pipeline — E4 and E5 +co-develop). This epic implements ADR-0004. + +**Features:** F4.1 typed intent model and closed vocabulary · F4.2 per-verb arg schema +validation · F4.3 `gitops.open-pr` (first write) · F4.4 signed intent dispatch · F4.5 per-spoke +local allowlist re-validation and local execution · F4.6 dry-run-first execution · F4.7 +live-mutation verbs. + +### F4.1 — Typed intent model and closed vocabulary (fail-safe allowlist) + +**What it is.** The intent as the unit of the write path — an object with a verb drawn from a +closed vocabulary — enforced by a fail-safe allowlist rather than a fail-open denylist. + +**How it works.** +1. An intent carries `{id, workspace, actor, verb, targetSelector, args, justification, + evidenceRefs, signature}`. +2. `verb` must be in the closed vocabulary and have a registered, schema-validated handler; an + unknown verb is refused, not executed. +3. A CI test asserts every handler that reaches a write path is classified against the closed + vocabulary — a forgotten classification fails the build. +4. Adding any verb is an ADR-level change, never a routine edit. + +```mermaid +flowchart TD + A["Proposed intent {verb, targetSelector, args, ...}"] --> B{"verb in CLOSED_VOCAB?"} + B -- "no" --> R["Refuse (fail-safe)"] + B -- "yes" --> C{"registered, schema-validated handler exists?"} + C -- "no" --> R + C -- "yes" --> D["Enter PEP pipeline (E5)"] + E["CI: every write-path handler classified vs vocabulary?"] --> F{"any unclassified?"} + F -- "yes" --> G["Fail the build (not production)"] + F -- "no" --> H["Build passes"] +``` + +**Acceptance criteria.** +- Only vocabulary verbs with registered handlers can execute; unknown verbs are refused. +- The CI classification test fails the build if any write-path handler is unclassified. + +**Key risk / guardrail.** The predecessor's fail-open "denylist of one" made a forgotten +classification auto-executable. Guardrail: the model is a fail-safe allowlist and the CI test +proves nothing reaches a write path unclassified. + +### F4.2 — Per-verb arg schema validation + +**What it is.** Every verb's arguments are validated against a per-verb JSON schema, and +execution is structured API calls — never string interpolation, because there is no shell. + +**How it works.** +1. Each verb registers a JSON schema for its args (for example, `deployment.scale` requires a + valid replica count and a resolved target). +2. Args that fail the schema are refused before any execution. +3. Valid args are passed to a typed handler that makes structured API calls (Argo CD API, + Rollouts API, the scale subresource, a Git PR). +4. No argument is ever concatenated into a command string; the shell path does not exist. + +```mermaid +flowchart TD + A["Intent args for verb V"] --> B{"Args valid against V's JSON schema?"} + B -- "no" --> R["Refuse"] + B -- "yes" --> C["Typed handler for V"] + C --> D["Structured API call (Argo CD / Rollouts / scale subresource / Git PR)"] + D --> E["No string interpolation, no shell — ever"] +``` + +**Acceptance criteria.** +- Args are validated per-verb; invalid args are refused before execution. +- Execution is structured API calls with no shell and no string interpolation. + +**Key risk / guardrail.** String interpolation into a command was the predecessor's RCE path. +Guardrail: verbs map to typed API calls only; there is no shell to inject into, and schema +validation rejects malformed args up front. + +### F4.3 — `gitops.open-pr` (the first and safest write) + +**What it is.** The first write Sith ships: opening a pull request on a target repo — a proposal +a human merges, requiring zero new standing trust and no cluster mutation. + +**How it works.** +1. An operator (or MCP client) proposes `gitops.open-pr` with a target repo/branch and the change + (for example, bump replicas for `web`). +2. The intent passes the full PEP pipeline (E5) and, on allow, the hub uses a Git credential held + via KMS envelope (E3) with the narrowest scope that can open a PR — no direct-push credential. +3. The change is opened as a PR; nothing is applied to any cluster. +4. A human reviews and merges; the merge (via GitOps) is what eventually changes state, keeping + Sith out of the reconcile loop. + +```mermaid +sequenceDiagram + autonumber + participant C as Client (UI / MCP) + participant PEP as PEP pipeline (E5) + participant V as KMS-envelope Git credential (E3) + participant REPO as Target repo + participant H as Human reviewer + C->>PEP: propose gitops.open-pr {repo, branch, change, justification} + PEP->>PEP: authn -> tenant -> role -> verb -> args -> scope -> PDP -> audit + PEP->>V: get narrow-scope Git credential (open-PR only) + V-->>PEP: short-scoped credential + PEP->>REPO: open pull request (no cluster mutation) + REPO-->>PEP: PR URL + PEP-->>C: PR opened (proposal) + H->>REPO: review + merge (GitOps applies later) + Note over PEP: proposed + executed both audited + decision-ledgered +``` + +**Acceptance criteria.** +- A `gitops.open-pr` intent flows end-to-end and opens a real PR. +- Zero cluster credentials reach the center or any AI at any point; the Git credential is + narrow-scope and KMS-protected. + +**Key risk / guardrail.** A broad Git credential (direct push) would let Sith change state +without human review. Guardrail: the credential can only open a PR, not push to protected +branches; the human merge is the change gate, consistent with GitOps orthodoxy. + +### F4.4 — Signed intent dispatch + +**What it is.** The hub signs every dispatched intent and sends it per target, wave-ordered, down +the same reverse tunnel — so each spoke can verify integrity before acting. + +**How it works.** +1. After the PEP allows an intent, the hub requests a signature from the KMS/HSM signing key + (E3). +2. The signed intent is dispatched to each target spoke over the `cluster-proxy` tunnel. +3. Dispatch is per target and wave-ordered (E5): later waves wait for earlier ones and their + gates. +4. Each spoke receives a signed intent it can independently verify (F4.5). + +```mermaid +sequenceDiagram + autonumber + participant PEP as PEP (hub) + participant KMS as KMS/HSM signing key + participant SPa as Spoke agent A + participant SPb as Spoke agent B + PEP->>KMS: sign intent + KMS-->>PEP: signature + Note over PEP,SPb: dispatch is wave-ordered (E5), per target, over the reverse tunnel + PEP->>SPa: signed intent (wave 1) + SPa-->>PEP: outcome + PEP->>SPb: signed intent (wave 2, after wave-1 gate) + SPb-->>PEP: outcome +``` + +**Acceptance criteria.** +- Every dispatched intent is signed by the hub; dispatch is per target and wave-ordered. +- A spoke receives a verifiable signed intent, not a raw command. + +**Key risk / guardrail.** An unsigned or replayable dispatch could be forged or replayed. +Guardrail: intents are signed (integrity anchor) and carry an id for idempotency/dedupe (E5), so +a spoke rejects an unverifiable or duplicate dispatch. + +### F4.5 — Per-spoke local allowlist re-validation and local execution + +**What it is.** A spoke never blindly executes what the hub sends. It verifies the signature, +re-validates the intent against its own local allowlist, and executes with its own scoped local +identity — the second of two independent blast-radius bounds. + +**How it works.** +1. The spoke agent verifies the intent signature against the hub's public key. +2. It re-validates the verb and target against its own local allowlist and local RBAC — + independent of the hub's vocabulary. +3. If both pass, it executes using its own scoped local identity (not a hub-supplied + credential), doing a dry-run first where applicable (F4.6). +4. It returns a per-cluster outcome to the hub; a failed check is a local refusal. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Hub (dispatch) + participant SP as Sith spoke agent + participant SVC as Cluster-local service + PEP->>SP: signed intent + SP->>SP: verify signature + alt signature invalid + SP-->>PEP: refuse (bad signature) + else valid + SP->>SP: re-validate vs LOCAL allowlist + local RBAC + alt not locally allowed + SP-->>PEP: refuse (local allowlist) + else allowed + SP->>SVC: execute with SP's OWN scoped identity (dry-run first) + SVC-->>SP: result + SP-->>PEP: per-cluster outcome + end + end +``` + +**Acceptance criteria.** +- A spoke executes only after verifying the signature and passing its own local allowlist. +- Execution uses the spoke's own scoped identity, never a hub-held credential. + +**Key risk / guardrail.** If the spoke trusted the hub blindly, a hub compromise would be +"execute anything everywhere". Guardrail: the spoke's independent signature check plus local +allowlist plus local RBAC bound the damage of a forged or compromised dispatch to what that spoke +already permits. + +### F4.6 — Dry-run-first execution + +**What it is.** For every verb that supports it, a dry-run runs first and surfaces the plan/diff; +executing is a separate, explicit step. + +**How it works.** +1. On an allowed intent, the handler performs a dry-run against the target and produces a + plan/diff. +2. The plan/diff is surfaced to the proposer (and to any approval step). +3. Execution proceeds only on a separate explicit action, not automatically from the dry-run. +4. Verbs that cannot dry-run declare that, so the operator knows the plan is not previewable. + +```mermaid +flowchart TD + A["Allowed intent"] --> B{"Verb supports dry-run?"} + B -- "yes" --> C["Dry-run -> produce plan/diff"] + C --> D["Surface plan/diff to proposer / approver"] + D --> E{"Explicit execute step taken?"} + E -- "no" --> F["Stop (no change)"] + E -- "yes" --> G["Execute"] + B -- "no" --> H["Declare non-previewable, require explicit execute"] + H --> E +``` + +**Acceptance criteria.** +- Every dry-run-capable verb previews a plan/diff before any change. +- Execution requires a separate explicit step; a dry-run never auto-executes. + +**Key risk / guardrail.** Auto-executing from a plan removes the human's last look. Guardrail: +execute is always a distinct, explicit step separated from the dry-run. + +### F4.7 — Live-mutation verbs (behind full policy) + +**What it is.** The verbs that change cluster state — `argocd.sync|rollback`, +`rollout.promote|abort`, `deployment.scale|restart` — enabled per workspace only after the PR +path is proven, and only behind the full policy layer. + +**How it works.** +1. Live verbs stay disabled until `gitops.open-pr` is proven end-to-end for a workspace. +2. When enabled, each still passes the complete PEP pipeline, is signed, re-validated locally, and + dry-run-first. +3. Fan-out for these verbs is governed by E5 (env gates, wave ordering, partial-failure/rollback, + abstention). +4. `exec` and free-form `apply` remain permanently excluded — live mutation never means arbitrary + mutation. + +```mermaid +flowchart TD + A["Request a live-mutation verb (argocd.sync, rollout.promote, deployment.scale, ...)"] --> B{"PR path proven for this workspace?"} + B -- "no" --> R["Refuse (not yet enabled)"] + B -- "yes" --> C["Full PEP pipeline (E5) + signed dispatch (F4.4)"] + C --> D["Spoke re-validate + dry-run + execute (F4.5/F4.6)"] + D --> E["Fan-out governed: waves, gates, rollback, abstention (E5)"] + F["exec / free-form apply / secret / RBAC mutation"] --> G["Permanently excluded — not expressible"] +``` + +**Acceptance criteria.** +- Live verbs are enabled per workspace only after the PR path is proven, and always ride the full + policy layer. +- `exec` and free-form `apply` remain impossible at every phase. + +**Key risk / guardrail.** Enabling live mutation broadly and early would expose real blast radius +before governance is proven. Guardrail: PR-first per workspace, full policy on every live verb, +and the permanent exclusions hold regardless of phase. + +### E4 exit criteria + +- The write path accepts only closed-vocabulary verbs with registered, schema-validated handlers; + the CI classification test is green. +- `gitops.open-pr` flows end-to-end and opens a real PR with zero cluster credentials in the + center or the AI. +- Every dispatched intent is signed; each spoke independently verifies and re-validates against + its own allowlist and executes with its own identity. +- Dry-run precedes execution for every capable verb; live-mutation verbs are gated behind the PR + proof and the full policy layer. +- No shell, no free-form apply, no secret/RBAC mutation exists anywhere in the path. + +## E5 — Policy federation and governance + +**Goal:** govern the fan-out of a single intent to N clusters — one ordered enforcement pipeline, +Ardur as the policy decision point and identity broker, environment gates and multi-approver +flows, wave/canary ordering with a gate per wave, partial-failure and auto-rollback, idempotency, +and honest abstention when the fleet view is incomplete. + +**Phase:** P1 (the policy-hook seam and pipeline shape, allowing reads) → P2 (Ardur PDP + identity +broker on the first write) → P3 (the full fan-out reasoning). **Depends on:** E1 (identity), E2 +(fleet model and staleness), and co-develops with E4 (every intent rides this pipeline). This is +the genuinely novel, hard part of Sith and implements ADR-0004/ADR-0005. + +**Features:** F5.1 PEP enforcement pipeline · F5.2 policy-hook seam · F5.3 Ardur PDP integration · +F5.4 Ardur scoped-identity broker · F5.5 environment gates and multi-approver · F5.6 wave/canary +ordering with a gate per wave · F5.7 partial-failure semantics and auto-rollback · F5.8 +federation-specific abstention · F5.9 elicited per-action approval bound to an arg-hash. + +### F5.1 — PEP enforcement pipeline + +**What it is.** The single ordered gate every intent passes — from the UI or the MCP server +alike — with no privileged back-door path. Fail-safe: anything not explicitly permitted is +refused. + +**How it works.** The pipeline runs in this order, and any step can refuse: +1. authn from the signed token (never headers); +2. workspace membership; +3. role gate; +4. closed verb vocabulary; +5. arg schema validation; +6. tenant-scoped target resolution (only within the actor's workspace); +7. Ardur PDP query (fan-out aware); +8. elicited approval (per-action, arg-hash bound); +9. scoped identity mint (ceiling below the human); +10. caps/budgets (max clusters per intent, rate limits); +11. signed, wave-ordered dispatch (spoke re-validates independently); +12. audit + decision ledger, always. + +```mermaid +flowchart TD + A["Intent (UI or MCP — same path)"] --> B["authn: signed token, not headers"] + B --> C["workspace membership"] + C --> D["role gate"] + D --> E["closed verb vocabulary"] + E --> F["arg schema validation"] + F --> G["tenant-scoped target resolution"] + G --> H["Ardur PDP query (fan-out aware)"] + H --> I["elicited approval (arg-hash bound)"] + I --> J["scoped identity mint (ceiling < human)"] + J --> K["caps / budgets"] + K --> L["signed, wave-ordered dispatch"] + L --> M["audit + decision ledger"] + B -. "any step may refuse" .-> R["Refuse (fail-safe) + audit"] + H -. "deny" .-> R + I -. "missing" .-> R +``` + +**Acceptance criteria.** +- Every intent, from UI or MCP, passes the same ordered pipeline; there is no privileged path. +- Any step can refuse; a refusal is fail-safe and audited. + +**Key risk / guardrail.** A back-door path (an agent route that skips a gate) would collapse the +whole model. Guardrail: the MCP server and UI are both thin clients onto this one pipeline; +tests assert no write path bypasses it. + +### F5.2 — Policy-hook seam (built early) + +**What it is.** The policy hook at the `executeIntent` boundary, present from Phase 1 returning +"allow" for reads, shaped so Ardur drops in for Phase 2 writes without re-architecture. + +**How it works.** +1. In P1, reads flow through the PEP and the policy hook, which returns allow, and are audited. +2. The hook's interface is shaped for a real PDP from the start: it can return allow, deny, or + require-approval(s). +3. In P2, the hook is wired to Ardur; the surrounding pipeline does not change. +4. This makes the seam a stable boundary rather than a later rewrite. + +```mermaid +flowchart TD + A["Read (P1) or intent (P2+)"] --> B["PEP reaches executeIntent boundary"] + B --> C["Policy hook"] + C --> D{"Phase?"} + D -- "P1 (reads)" --> E["Return allow, audit"] + D -- "P2+ (writes)" --> F["Delegate to Ardur PDP (F5.3)"] + F --> G["allow / deny / require-approval"] + E --> H["Continue pipeline"] + G --> H +``` + +**Acceptance criteria.** +- In P1, every read flows through the PEP and policy hook and is audited. +- The hook interface supports allow/deny/require-approval and accepts Ardur without pipeline + changes. + +**Key risk / guardrail.** Building the write pipeline first and retrofitting policy later invites +an ungoverned interim. Guardrail: the seam exists from day one so the enforcement shape is fixed +before any write is possible. + +### F5.3 — Ardur PDP integration + +**What it is.** At the `executeIntent` boundary, the PEP asks Ardur, for every intent, whether +this actor may issue this verb on these resolved targets in this workspace right now — and gets +back allow, deny, or require-approval(s), fan-out aware. + +**How it works.** +1. The PEP sends Ardur the actor, verb, resolved targets, and workspace. +2. Ardur evaluates versioned, per-tenant, fan-out-aware policy (env gates, multi-approver, caps). +3. It returns allow / deny / require-approval(s) and records the reasons in its decision-ledger + (E6). +4. The PEP acts on the verdict; a deny is a fail-safe refusal. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Sith PEP + participant PDP as Ardur PDP + participant LEDG as Decision-ledger (E6) + PEP->>PDP: may {actor} run {verb} on {resolved targets} in {workspace} now? + PDP->>PDP: evaluate versioned, per-tenant, fan-out-aware policy + PDP->>LEDG: record reasons (why-allowed / why-denied) + PDP-->>PEP: allow / deny / require-approval(s) + alt deny + PEP->>PEP: refuse (fail-safe) + audit + else allow or require-approval + PEP->>PEP: continue pipeline (approval if required) + end +``` + +**Acceptance criteria.** +- Every intent is adjudicated by Ardur with a real verdict; the reasons land in the + decision-ledger. +- A deny results in a fail-safe refusal. + +**Key risk / guardrail.** Hardcoding "which verbs need approval" in Sith would drift from policy +and be unauditable. Guardrail: the decision is Ardur's versioned per-tenant policy, recorded with +reasons, so the "why" is external, explicit, and reviewable. + +### F5.4 — Ardur scoped-identity broker + +**What it is.** Ardur mints the short-lived, per-action, scoped execution identity for each +allowed action, so the AI/agent never holds a cluster credential and its ceiling is strictly +below the human's. + +**How it works.** +1. On allow, the PEP asks Ardur to mint an execution identity for this specific action. +2. The identity is short-lived and scoped to exactly the verb's needs — and capped below the + human actor's own privileges. +3. This complements OCM `managed-serviceaccount` on the spoke side: brokered identity governs the + action; the spoke still executes with its own local identity (E4). +4. The identity expires after the action; nothing long-lived is issued to an agent. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Sith PEP + participant IDB as Ardur identity broker + participant SP as Spoke agent + PEP->>IDB: mint scoped, short-lived identity for THIS action + IDB-->>PEP: per-action identity (ceiling below human, expires) + PEP->>SP: dispatch (identity governs the action) + SP->>SP: execute with SP's OWN local identity (re-validated) + Note over PEP,SP: agent never holds a cluster credential, identity is per-action and expiring +``` + +**Acceptance criteria.** +- Each allowed action uses a short-lived, per-action identity scoped below the human actor. +- No agent ever holds a standing cluster credential. + +**Key risk / guardrail.** A long-lived or over-scoped brokered identity would leak standing power +to an agent. Guardrail: identities are per-action, expiring, and ceiling-capped below the human; +the spoke additionally executes with its own local identity. + +### F5.5 — Environment gates and multi-approver + +**What it is.** Environment-aware gates: `prod` never auto-executes, prod requires N-person +approval, multi-cluster prod fan-out requires multiple approvers, and a max-clusters-per-intent +ceiling applies. + +**How it works.** +1. The resolved targets' environment labels (from the fleet model) determine the gate. +2. Any prod target forces approval; a single approver is never enough for multi-cluster prod + fan-out. +3. A max-clusters-per-intent ceiling caps blast radius regardless of environment. +4. Approvers must be distinct from the proposer (separation of duties, F1.3). + +```mermaid +flowchart TD + A["Resolved targets + env labels"] --> B{"Any prod target?"} + B -- "no" --> C{"Cluster count <= ceiling?"} + B -- "yes" --> D["Require approval (never auto)"] + D --> E{"Multi-cluster prod fan-out?"} + E -- "yes" --> F["Require multiple distinct approvers"] + E -- "no" --> G["Require one approver (distinct from proposer)"] + C -- "yes" --> H["Proceed (subject to PDP)"] + C -- "no" --> R["Refuse: exceeds max-clusters-per-intent"] + F --> C + G --> C +``` + +**Acceptance criteria.** +- No prod target ever auto-executes; multi-cluster prod requires multiple distinct approvers. +- An intent exceeding the max-clusters ceiling is refused. + +**Key risk / guardrail.** A mislabeled environment could route a prod cluster through a +non-prod gate. Guardrail: gates read env labels from the tenant-scoped fleet model, the ceiling +applies regardless of labels, and prod-without-approval is impossible by construction. + +### F5.6 — Wave/canary ordering with a gate per wave + +**What it is.** Sith plans a fan-out as ordered waves (for example dev → staging → one canary prod +→ health-gate → the rest), where each wave is separately gated and a health check runs between +waves. + +**How it works.** +1. The action-federation service plans the target set into ordered waves. +2. Each wave has its own gate; no wave proceeds without passing its gate. +3. Between waves, a health check evaluates the just-completed wave before the next begins. +4. A failed health check or gate stops progression (and triggers F5.7). + +```mermaid +flowchart TD + A["Plan target set into ordered waves"] --> B["Wave 1: dev"] + B --> C{"Wave gate + health check pass?"} + C -- "no" --> S["Stop (invoke partial-failure handling F5.7)"] + C -- "yes" --> D["Wave 2: staging"] + D --> E{"Wave gate + health check pass?"} + E -- "no" --> S + E -- "yes" --> F["Wave 3: one canary prod"] + F --> G{"Canary healthy?"} + G -- "no" --> S + G -- "yes" --> H["Wave 4: remaining prod"] +``` + +**Acceptance criteria.** +- A fan-out runs as ordered waves; each wave has its own gate and a health check between waves. +- No wave proceeds without passing its gate; a failed gate stops progression. + +**Key risk / guardrail.** Fanning out to everything at once is the largest self-inflicted blast +radius. Guardrail: waves with per-wave gates and inter-wave health checks bound how much can +change before a problem is caught. + +### F5.7 — Partial-failure semantics and auto-rollback + +**What it is.** Stop-on-first-failure with auto-rollback of the failed wave, plus +idempotency/dedupe so a retry cannot double-apply. + +**How it works.** +1. If a target in a wave fails, progression stops — later waves do not run. +2. The failed wave is automatically rolled back (using the paired verb where applicable, for + example `rollout.abort` / `argocd.rollback`). +3. Each intent and target carries an idempotency key; a retry with the same key is deduped rather + than re-applied. +4. The outcome (stopped, rolled back, per-cluster results) is recorded. + +```mermaid +flowchart TD + A["Executing a wave"] --> B{"Any target failed?"} + B -- "no" --> C["Wave succeeded -> next wave (F5.6)"] + B -- "yes" --> D["Stop: do not run later waves"] + D --> E["Auto-rollback the failed wave (paired verb)"] + E --> F["Record per-cluster outcomes"] + G["Retry with same idempotency key"] --> H{"Already applied?"} + H -- "yes" --> I["Dedupe: no double-apply"] + H -- "no" --> A +``` + +**Acceptance criteria.** +- A forced mid-rollout failure stops progression and auto-rolls-back that wave. +- A retry with the same idempotency key does not double-apply. + +**Key risk / guardrail.** A retry after a partial failure could double-apply and compound damage. +Guardrail: idempotency keys dedupe retries, and stop-on-failure plus auto-rollback bound a partial +failure to the failed wave. + +### F5.8 — Federation-specific abstention + +**What it is.** When the targeted fleet set is incomplete or stale, Sith refuses fleet-wide action +and says so honestly — a first-class, logged outcome unique to a federated world. + +**How it works.** +1. Before a fan-out, Sith checks the freshness/coverage of the targeted set (from E2/F2.5). +2. If any targeted cluster is stale beyond threshold or not visible, it abstains rather than + acting on a partial view. +3. It returns an honest message naming the gap (for example, "37/40 clusters visible; 3 stale + >10m — I will not issue a fleet sync until they report"). +4. The abstention is logged as a first-class outcome, not an error. + +```mermaid +flowchart TD + A["Fleet-wide intent over targeted set"] --> B["Check coverage + freshness of targeted set (F2.5)"] + B --> C{"All targeted clusters visible and fresh?"} + C -- "yes" --> D["Proceed to fan-out (F5.6)"] + C -- "no" --> E["Abstain: refuse fleet-wide action"] + E --> F["Return honest message: N/M visible, K stale >threshold"] + F --> G["Log abstention as a first-class outcome"] +``` + +**Acceptance criteria.** +- With one spoke made stale, a fleet-wide intent abstains with a correct, honest message. +- The abstention is logged as a first-class outcome, not an error. + +**Key risk / guardrail.** Acting on a partial view (guessing about dark clusters) is more +dangerous than not acting. Guardrail: abstention is mandatory on incomplete/stale coverage and is +logged, so "I won't act" is visible and defensible rather than silent best-effort. + +### F5.9 — Elicited per-action approval bound to an arg-hash + +**What it is.** A required approval that is per-action, non-reusable, and bound to a hash of the +resolved args — so an actor (human or agent) cannot approve one thing and then swap the +arguments. + +**How it works.** +1. When the PDP requires approval, the PEP computes a hash of the fully resolved args and target + set. +2. The approval request is presented (elicited) bound to that hash. +3. The approver approves the specific hashed action; the approval is single-use. +4. Before dispatch, the PEP re-checks that the args still match the approved hash; any mismatch + refuses. + +```mermaid +sequenceDiagram + autonumber + participant PEP as PEP + participant AP as Approver (human) + PEP->>PEP: compute hash of resolved args + targets + PEP->>AP: elicit approval bound to arg-hash + AP-->>PEP: approval (single-use, for this hash) + PEP->>PEP: before dispatch, re-check args hash == approved hash + alt hash mismatch (args changed) + PEP->>PEP: refuse (approve-then-swap blocked) + else match + PEP->>PEP: dispatch (F4.4) + end +``` + +**Acceptance criteria.** +- Approvals are per-action, single-use, and bound to the resolved-args hash. +- Changing the args after approval invalidates it (approve-then-swap is blocked). + +**Key risk / guardrail.** Approve-then-swap (approve a benign action, then change args) is the +classic agent bypass. Guardrail: the approval is bound to an arg-hash re-checked at dispatch, so a +valid signature and a valid approval are both necessary but neither is sufficient if the args +changed. + +### E5 exit criteria + +- Every intent, UI or MCP, passes one ordered PEP pipeline with no privileged path; the policy + seam exists from P1. +- Ardur returns real allow/deny/require-approval verdicts, records reasons, and mints per-action + scoped identities below the human ceiling. +- Prod never auto-acts; multi-cluster prod needs multiple distinct approvers; a max-clusters + ceiling holds. +- A wave-ordered fan-out runs with a gate per wave and inter-wave health checks; a mid-rollout + failure stops and auto-rolls-back that wave; retries dedupe. +- A stale targeted set produces a correct, logged abstention; approvals are arg-hash-bound and + single-use. + +## E6 — Audit and decision ledger + +**Goal:** keep a complete, tamper-evident record of what happened (Sith's audit-log) and why it +was allowed (Ardur's decision-ledger), together forming one agent-action record for humans and +agents alike. + +**Phase:** P1 (reads are audited through the seam) → P2 (the full ledger is populated when the +first write flows). **Depends on:** E1 (tenancy), E5 (the PEP is where entries are produced). The +two records are deliberately separate and complementary (ARCHITECTURE §5, §8). + +**Features:** F6.1 audit-log (what-happened) · F6.2 decision-ledger (why-allowed) · F6.3 +tamper-evidence and append-only storage · F6.4 unified action record with query/export. + +### F6.1 — Audit-log (what-happened) + +**What it is.** Sith's record of every phase of every action — proposed, approved, dry-run, +executed — for reads and writes, always. + +**How it works.** +1. As an intent moves through the pipeline, each phase writes an audit entry (proposed → approved + → dry-run → executed, plus refused/abstained where they occur). +2. Each entry carries the intent id, phase, workspace, actor, and a what-happened detail. +3. Reads are audited too (through the P1 seam), so the log covers the whole governed surface. +4. Entries are written on the same path as enforcement, so there is no unlogged action. + +```mermaid +flowchart TD + A["Intent / read moves through PEP"] --> B["Phase: proposed -> audit entry"] + B --> C["Phase: approved -> audit entry"] + C --> D["Phase: dry-run -> audit entry"] + D --> E["Phase: executed -> audit entry"] + A --> F["Refused / abstained -> audit entry"] + B --> G[("Audit-log: intent_id, phase, actor, workspace, detail, at")] + C --> G + D --> G + E --> G + F --> G +``` + +**Acceptance criteria.** +- Every phase of every action (and every read) produces an audit entry. +- No action reaches a spoke without a corresponding audit trail. + +**Key risk / guardrail.** An action path that skips logging would create a blind spot in exactly +the highest-risk surface. Guardrail: audit writes are on the enforcement path itself, so an +un-audited action is not reachable. + +### F6.2 — Decision-ledger (why-allowed) + +**What it is.** Ardur's record of why each intent was allowed, denied, or sent to approval — +complementing the audit-log's what-happened with the reasons behind the verdict. + +**How it works.** +1. When Ardur adjudicates an intent (F5.3), it records the verdict and the reasons + (which policy, which gate, which approvers required). +2. The decision is keyed to the same intent id as the audit-log. +3. The allow decision is bound to the hash of the resolved args (F5.9), so the "why" is tied to + the exact action approved. +4. Together, the audit-log and decision-ledger reconstruct both what happened and why it was + permitted. + +```mermaid +sequenceDiagram + autonumber + participant PDP as Ardur PDP (F5.3) + participant LEDG as Decision-ledger + participant AUD as Audit-log (F6.1) + PDP->>LEDG: record verdict + reasons (why-allowed / why-denied), arg-hash bound + PDP-->>AUD: same intent_id links the two records + Note over LEDG,AUD: why-allowed (Ardur) + what-happened (Sith) = complete action record +``` + +**Acceptance criteria.** +- Every verdict records its reasons in the decision-ledger, keyed to the intent id. +- The allow decision is bound to the resolved-args hash. + +**Key risk / guardrail.** A verdict without recorded reasons is unauditable ("it said yes, but +why?"). Guardrail: reasons are recorded with each verdict and bound to the arg-hash, so the "why" +is specific and reviewable. + +### F6.3 — Tamper-evidence and append-only storage + +**What it is.** Both records are append-only and tamper-evident, so an attacker who reaches the +store cannot silently rewrite history to hide an action. + +**How it works.** +1. Entries are append-only — no in-place update or delete on the enforcement path. +2. Each entry is chained to the prior entry's hash, so any later edit breaks the chain. +3. An integrity verification can walk the chain and detect any altered or removed entry. +4. This makes the audit-log and decision-ledger forensic assets even under partial compromise. + +```mermaid +flowchart TD + A["New audit / decision entry"] --> B["Compute hash over entry + previous entry's hash"] + B --> C[("Append-only store (chained hashes)")] + C --> D["Integrity check walks the chain"] + D --> E{"Any entry altered or missing?"} + E -- "yes" --> F["Tampering detected (chain broken)"] + E -- "no" --> G["History intact"] +``` + +**Acceptance criteria.** +- Entries are append-only and hash-chained; an altered or removed entry breaks verification. +- An integrity check can detect tampering after the fact. + +**Key risk / guardrail.** Tampering hides an attack (S-class: forensics destroyed). Guardrail: the +hash chain makes any edit detectable, so even a store-level compromise cannot silently rewrite +what happened. + +### F6.4 — Unified action record with query/export + +**What it is.** The ability to correlate the audit-log and decision-ledger by intent id into one +view, and to export it for compliance and incident review — tenant-scoped. + +**How it works.** +1. A query joins audit entries and decision entries on intent id to show the full lifecycle of an + action. +2. Queries and exports are tenant-scoped (E1), so one workspace never sees another's records. +3. An export produces a portable record (what happened, when, who, why allowed) for compliance + owners. +4. This answers the security/compliance question: prove what operators — human and agent — were + allowed to do and did do across the fleet. + +```mermaid +flowchart TD + A["Query by intent_id (tenant-scoped)"] --> B["Join audit-log (what) + decision-ledger (why)"] + B --> C["Unified lifecycle: proposed -> approved -> executed + reasons"] + C --> D{"Export requested?"} + D -- "yes" --> E["Portable compliance record (workspace-scoped)"] + D -- "no" --> F["Show in console / API"] +``` + +**Acceptance criteria.** +- Audit and decision records correlate by intent id into one lifecycle view. +- Query/export is tenant-scoped and produces a portable compliance record. + +**Key risk / guardrail.** A cross-tenant leak in an export would be a serious confidentiality +breach. Guardrail: query and export inherit E1's tenant scoping and DB RLS, so a record is only +ever visible within its own workspace. + +### E6 exit criteria + +- Every phase of every action and read produces an audit entry; no unlogged action reaches a + spoke. +- Every verdict records its reasons in the decision-ledger, keyed to intent id and bound to the + arg-hash. +- Both records are append-only and hash-chained; tampering is detectable. +- Audit and decision correlate into one tenant-scoped lifecycle view with a portable export. + +## E7 — Governed MCP server + +> **Reshape note:** the MCP **read** tools (F7.1) ship early, in **Phase L** with the local +> client (E11) — the shadow-MCP lesson makes the sanctioned read path a day-0 requirement. The +> **write** tools (F7.2–F7.5) stay gated behind the governed write path (P2 `gitops.open-pr`, +> then P3 fan-out). Everything still rides the same PEP; nothing about the enforcement changes. + +**Goal:** expose the federated read and action surface as an MCP server so external agents +(Claude Code, Codex, kagent) become clients that inherit exactly the governance a human has — the +same PEP, the same PDP, the same closed vocabulary, the same audit — a governed MCP gateway to the +whole fleet. + +**Phase:** P3 · **Depends on:** E2 (reads), E4 (write vocabulary), E5 (PEP/PDP), E6 (audit + +ledger). MCP tool annotations shipped in the 2025-03-26 spec and are hints, not guarantees, so +enforcement is server-side; Elicitation shipped in the 2025-06-18 spec as the native primitive +for human-in-the-loop approval. Implements ADR-0005. + +**Features:** F7.1 MCP read tools · F7.2 MCP write tools mapped 1:1 to the closed vocabulary · +F7.3 elicitation-based approval on writes · F7.4 external agent as governed client · F7.5 AI +safety rules. + +### F7.1 — MCP read tools + +**What it is.** Read tools (`fleet.inventory`, `fleet.health`, `fleet.correlate`, +`fleet.cve-search`) carrying `readOnlyHint: true`, hitting the fleet model, scoped to the caller's +workspace. + +**How it works.** +1. An external agent calls a read tool over MCP. +2. The MCP layer is a thin client onto the same PEP; the read is tenant-scoped to the caller's + workspace. +3. The fleet model (E2) answers, including cross-cluster correlation and CVE search. +4. The read is audited like any other (E6); no gate beyond tenant scope is needed for reads. + +```mermaid +sequenceDiagram + autonumber + participant AG as External agent (MCP client) + participant MCP as Sith MCP server + participant PEP as PEP (tenant scope) + participant FM as Fleet model (E2) + AG->>MCP: call fleet.correlate (readOnlyHint: true) + MCP->>PEP: same path as UI — resolve workspace scope + PEP->>FM: query within caller's workspace + FM-->>PEP: cross-cluster answer (stale flagged) + PEP-->>MCP: result (audited) + MCP-->>AG: result +``` + +**Acceptance criteria.** +- Read tools return workspace-scoped fleet answers, including correlation and CVE search. +- Reads are audited and carry `readOnlyHint`. + +**Key risk / guardrail.** A read tool that ignored tenant scope would leak cross-tenant fleet +data. Guardrail: MCP reads go through the same PEP tenant scoping and DB RLS as the UI — the MCP +layer has no privileged data path. + +### F7.2 — MCP write tools mapped 1:1 to the closed vocabulary + +**What it is.** Write tools that map one-to-one to the closed verb vocabulary, carry +`destructiveHint: true` and correct `idempotentHint`, and are enforced server-side because +annotations are only hints. + +**How it works.** +1. Each write tool (`intent.gitops-open-pr` first, later `intent.argocd-sync`, + `intent.rollout-promote`, `intent.deployment-scale`) corresponds to exactly one vocabulary + verb. +2. The tool declares annotations for client UX, but the server does not trust them — it runs the + full PEP pipeline (E5) regardless. +3. There is no write tool outside the closed vocabulary; there is no generic "apply" or "exec" + tool to call. +4. `intent.gitops-open-pr` is the only write enabled first; others follow per-workspace after the + PR path is proven. + +```mermaid +flowchart TD + A["MCP write tool call (e.g. intent.gitops-open-pr)"] --> B["Annotations (destructiveHint/idempotentHint) = client UX hints only"] + B --> C["Server does NOT trust annotations"] + C --> D["Run full PEP pipeline (E5) — same as UI"] + D --> E{"1:1 with a closed-vocabulary verb?"} + E -- "no" --> R["No such tool / refuse"] + E -- "yes" --> F["Proceed under governance (approval, dispatch, audit)"] +``` + +**Acceptance criteria.** +- Write tools map 1:1 to closed-vocabulary verbs; there is no generic apply/exec tool. +- Enforcement is server-side; annotations are treated as hints only. + +**Key risk / guardrail.** A malicious or buggy client could mislabel a destructive tool as +read-only to dodge a confirmation. Guardrail: the server enforces at the PEP regardless of +annotations — the spec is explicit that annotations are not guarantees. + +### F7.3 — Elicitation-based approval on writes + +**What it is.** Write tools require Elicitation-based approval (the 2025-06-18 MCP primitive) +bound to a hash of the resolved args, and elicitation is never used to request secrets. + +**How it works.** +1. When a write needs approval, the server issues an `elicitation/create` request with a JSON + schema describing the approval, bound to the resolved-args hash (F5.9). +2. The client presents it to the user, who approves the specific action. +3. The approval is single-use; before dispatch the server re-checks the args hash. +4. Elicitation requests structured approval only — never credentials or other sensitive data, per + the spec's constraint. + +```mermaid +sequenceDiagram + autonumber + participant AG as MCP client (agent) + participant MCP as Sith MCP server + participant U as User + MCP->>MCP: compute resolved-args hash (F5.9) + MCP->>AG: elicitation/create {schema, bound to arg-hash} (never requests secrets) + AG->>U: present approval request + U-->>AG: approve (single-use, this hash) + AG-->>MCP: approval + MCP->>MCP: re-check args hash == approved hash + alt mismatch + MCP-->>AG: refuse (approve-then-swap blocked) + else match + MCP->>MCP: dispatch under governance + end +``` + +**Acceptance criteria.** +- Writes require single-use elicited approval bound to the resolved-args hash. +- Elicitation never requests credentials or sensitive data. + +**Key risk / guardrail.** Eliciting sensitive data, or a reusable approval, would create a leak or +a bypass. Guardrail: approvals are arg-hash-bound and single-use, and elicitation is limited to +structured approval — never a vehicle for secrets. + +### F7.4 — External agent as governed client + +**What it is.** Any external agent — Claude Code, Codex, kagent — is a client of the same +governance, with no privileged path: it gets exactly the governance a human does. + +**How it works.** +1. The agent connects as an MCP client and can call read and write tools. +2. Every call runs the same PEP pipeline and Ardur PDP as the UI; the agent identity ceiling is + strictly below the human's (F5.4). +3. Writes require the same elicited, arg-hash-bound approval; the agent never holds a cluster + credential. +4. Everything the agent does is audited and decision-ledgered identically to a human action. + +```mermaid +sequenceDiagram + autonumber + participant EXT as Claude Code / Codex / kagent + participant MCP as Sith MCP server + participant PEP as PEP + Ardur PDP + participant AUD as Audit + decision ledger + EXT->>MCP: read or write tool call + MCP->>PEP: SAME pipeline as UI (no privileged path) + PEP->>PEP: role/scope/verb/args/PDP/approval (ceiling below human) + PEP->>AUD: audit + decision-ledger (identical to human) + PEP-->>MCP: result / refusal + MCP-->>EXT: result / refusal +``` + +**Acceptance criteria.** +- An external MCP client issuing a read and a write is subject to identical governance (scope, + approval, audit). +- The agent holds no cluster credential and its ceiling is below the human's. + +**Key risk / guardrail.** An agent path that bypassed the PEP would be the shortest route from a +prompt to a fleet action. Guardrail: the MCP server is a thin client onto the one PEP; there is no +agent-only route, and the agent inherits every gate a human faces. + +### F7.5 — AI safety rules + +**What it is.** The behavioral rules baked into the agent surface: ground-or-abstain, evidence +before a write proposal, per-actor token/action budgets, and write proposals rate-limited +separately from reads. + +**How it works.** +1. Any statement about live state must be backed by a tool result or flagged as general knowledge + (ground-or-abstain). +2. A write may be proposed only from an evidence-citing chain; low confidence yields "here's what + I'd check", not a write. +3. Per-tenant/per-actor token and action budgets bound how much an agent can do. +4. Write proposals are rate-limited separately from (and more tightly than) reads. + +```mermaid +flowchart TD + A["Agent wants to act / assert"] --> B{"Claim backed by a tool result?"} + B -- "no" --> C["Flag as general knowledge or abstain"] + B -- "yes" --> D{"Evidence sufficient for a write?"} + D -- "no" --> E["Propose checks, not a write"] + D -- "yes" --> F{"Within token/action budget + write rate-limit?"} + F -- "no" --> G["Refuse / defer (budget/limit)"] + F -- "yes" --> H["Propose write (still gated by PEP + approval)"] +``` + +**Acceptance criteria.** +- Live-state claims are grounded in tool results or flagged; low-confidence never yields a write + proposal. +- Per-actor budgets and separate write rate-limits are enforced. + +**Key risk / guardrail.** An ungrounded or runaway agent could flood the write surface with +plausible-but-wrong proposals. Guardrail: ground-or-abstain plus separate, tighter write +rate-limits and budgets bound both the quality and the volume of what an agent can propose — and +every proposal still faces the full PEP. + +### E7 exit criteria + +- Read tools return workspace-scoped answers with `readOnlyHint`; write tools map 1:1 to the + closed vocabulary with server-side enforcement. +- Writes require single-use, arg-hash-bound elicited approval; elicitation never requests secrets. +- An external MCP client is governed identically to a human (same PEP/PDP, ceiling below human, no + credential, fully audited). +- AI safety rules (ground-or-abstain, evidence-gated writes, budgets, separate write rate-limits) + are enforced. + +## E8 — Operator console (UI) + +> **Reshape note:** the console is **one web frontend** served by both `sith ui` (local, +> single-user, kubeconfig-direct — the day-0 "fleet IDE") and `sith hub` (multi-user, governed). +> The local fleet view (E11) and this hub console render the same source-abstract fleet model. + +**Goal:** a thin, unprivileged operator console — fleet view, a workspace/cluster/service picker, +intent proposal with plan preview, multi-approver approval, and run/wave status — that is a +client of the governed API with no privileged path. + +**Phase:** P1 (a thin read view) → P3 (proposal, approval, and wave status). **Depends on:** E2 +(reads), E5 (proposal/approval flows), E6 (status and history). Per ADR-0002 the UI is +deliberately minimal; the product's value is the governed API, and the UI has exactly the +governance the MCP surface does. + +**Features:** F8.1 fleet view · F8.2 workspace/cluster/service picker · F8.3 intent proposal UX · +F8.4 multi-approver approval UX · F8.5 run/wave status view. + +### F8.1 — Fleet view + +**What it is.** A view of inventory and health across the workspace's clusters, with freshness +badges so coverage gaps are visible. + +**How it works.** +1. The view calls the read API (E2), which is tenant-scoped, and renders inventory and health per + cluster. +2. Each cluster and record shows a freshness badge; stale clusters are marked (F2.5). +3. Cross-cluster correlation results (for example "clusters where `payments` is Degraded") render + as a single fleet-wide answer. +4. The view is read-only and holds no privileged path — it only shows what the API returns for the + actor's workspace. + +```mermaid +flowchart TD + A["Operator opens fleet view"] --> B["Call read API (tenant-scoped, E2)"] + B --> C["Render inventory + health per cluster"] + C --> D["Freshness badge per cluster (stale flagged, F2.5)"] + C --> E["Cross-cluster correlation shown as one answer"] + D --> F["Coverage gaps visible, not hidden"] +``` + +**Acceptance criteria.** +- The fleet view renders tenant-scoped inventory/health with per-cluster freshness badges. +- Stale clusters are visibly flagged. + +**Key risk / guardrail.** A UI that hid staleness would present a false-complete picture. +Guardrail: freshness badges surface staleness in the view, mirroring the API's honesty about +coverage. + +### F8.2 — Workspace/cluster/service picker + +**What it is.** Tenant-scoped navigation to pick a workspace, then a cluster, then a service — +showing only what the actor is a member of. + +**How it works.** +1. The picker lists only the workspaces the actor is a member of (from signed-token claims, E1). +2. Selecting a workspace scopes everything downstream to it. +3. Within a workspace, the operator narrows to a cluster and then a service. +4. There is no way to select a workspace or cluster outside the actor's membership. + +```mermaid +flowchart TD + A["Picker opens"] --> B["List workspaces from actor's membership claims (E1)"] + B --> C["Select workspace -> scope everything to it"] + C --> D["Select cluster (within workspace)"] + D --> E["Select service (within cluster)"] + B --> F{"Workspace outside membership?"} + F -- "not shown / not selectable" --> B +``` + +**Acceptance criteria.** +- The picker shows only workspaces the actor belongs to; selection scopes all downstream views. +- No out-of-membership workspace or cluster is selectable. + +**Key risk / guardrail.** Exposing non-member workspaces in the picker would leak their existence. +Guardrail: the list derives from membership claims and is tenant-scoped server-side; the picker +cannot reach beyond it. + +### F8.3 — Intent proposal UX + +**What it is.** The flow where an operator proposes an intent — choosing a verb, a target +selector, and reviewing the dry-run plan/diff — submitted to the same governed API. + +**How it works.** +1. The operator picks a verb from the closed vocabulary and a target selector (resolved within the + workspace). +2. The UI requests a dry-run (F4.6) and shows the plan/diff before anything executes. +3. On submit, the proposal goes through the full PEP pipeline (E5) — the UI adds no privileged + path. +4. If the proposal requires approval, the UI reflects that it is pending, bound to the resolved + args. + +```mermaid +sequenceDiagram + autonumber + participant OP as Operator (UI) + participant API as Governed API + participant PEP as PEP pipeline (E5) + OP->>API: choose verb + target selector + API->>PEP: resolve target within workspace, dry-run (F4.6) + PEP-->>API: plan / diff + API-->>OP: show plan / diff before execute + OP->>API: submit proposal + API->>PEP: full pipeline (verb/args/scope/PDP/approval) + PEP-->>API: allowed / pending-approval / refused + API-->>OP: reflect status (bound to resolved args) +``` + +**Acceptance criteria.** +- An operator can propose a closed-vocabulary verb with a workspace-scoped target and preview the + plan/diff. +- Submission runs the full pipeline; the UI adds no privileged path. + +**Key risk / guardrail.** A UI that executed without a plan preview or that bypassed the pipeline +would undercut the safety model. Guardrail: dry-run precedes execute in the UI, and submission +goes through the same PEP as every other client. + +### F8.4 — Multi-approver approval UX + +**What it is.** The approval experience: an approver sees a pending action bound to its +resolved-args hash and approves it, with proposer and approver required distinct for gated +actions. + +**How it works.** +1. A gated intent appears in an approver's queue with its resolved args and the plan. +2. The approval is bound to the arg-hash (F5.9); the approver approves that specific action. +3. For multi-cluster prod, multiple distinct approvers are required (F5.5); the proposer cannot + self-approve. +4. On sufficient approvals, the action proceeds; the approval is single-use. + +```mermaid +sequenceDiagram + autonumber + participant AP as Approver (UI) + participant API as Governed API + participant PEP as PEP + API->>AP: show pending action + resolved args (arg-hash bound) + AP->>API: approve (must differ from proposer) + API->>PEP: register approval (single-use, this hash) + PEP->>PEP: enough distinct approvers? (multi-cluster prod) + alt sufficient + PEP->>PEP: proceed to dispatch + else insufficient + PEP-->>AP: still pending (await more approvers) + end +``` + +**Acceptance criteria.** +- An approver approves a specific arg-hash-bound action; the proposer cannot self-approve. +- Multi-cluster prod requires multiple distinct approvers before proceeding. + +**Key risk / guardrail.** Self-approval or a reusable approval would defeat separation of duties. +Guardrail: proposer/approver distinctness and single-use, arg-hash-bound approvals are enforced +server-side (F5.5/F5.9), not merely in the UI. + +### F8.5 — Run/wave status view + +**What it is.** A live view of a running fan-out: per-cluster outcomes, wave gates, rollback, and +any abstention message. + +**How it works.** +1. As a fan-out runs, the view shows each wave and each target's outcome (pending, succeeded, + failed). +2. Wave gates and inter-wave health checks are shown so the operator sees why the next wave has or + has not started. +3. A partial failure shows the stopped progression and the auto-rollback of the failed wave (F5.7). +4. An abstention shows the honest coverage message (for example "37/40 visible; 3 stale") rather + than a silent stop. + +```mermaid +flowchart TD + A["Fan-out running"] --> B["Show waves + per-cluster outcomes (pending/ok/failed)"] + B --> C["Show wave gates + inter-wave health checks"] + C --> D{"Partial failure?"} + D -- "yes" --> E["Show stop + auto-rollback of failed wave (F5.7)"] + D -- "no" --> F["Show progression through waves"] + A --> G{"Abstained?"} + G -- "yes" --> H["Show honest coverage message (F5.8)"] +``` + +**Acceptance criteria.** +- The view shows per-cluster outcomes, wave gates, rollback, and abstention messages in real time. +- A partial failure and its rollback are visible, and an abstention is shown honestly. + +**Key risk / guardrail.** A status view that showed only success/failure without the abstention or +rollback context would mislead the operator. Guardrail: the view surfaces gates, rollback, and the +abstention message directly from the run record (E6), so what the operator sees matches what +actually happened. + +### E8 exit criteria + +- A thin, unprivileged console renders tenant-scoped fleet view with freshness badges and a + membership-bounded picker. +- Operators propose closed-vocabulary intents with a plan preview through the same governed API. +- Approval UX enforces distinct proposer/approver and arg-hash-bound, single-use approvals + server-side. +- Run/wave status shows per-cluster outcomes, gates, rollback, and abstention truthfully. + +## E9 — Deployment and packaging + +> **Reshape note:** three things are day-one, not later. (1) **Multi-arch images** +> (`linux/amd64`+`arm64`) and **registry-relocatable** references — required for China/regulated +> estates and for arm64 laptops. (2) **cosign-signed releases + SLSA L2 provenance + SBOM** from +> the first tag. (3) The **local client** (E11) ships as a **single binary via `brew`/package +> managers** — that install path is the adoption funnel and belongs to this epic. + +**Goal:** package the hub as a Helm chart and the spoke agent as an OCM addon, support light and +heavy deployment profiles and air-gapped/on-prem installs, and define an upgrade path with an +ADR-gated addon version policy. + +**Phase:** M0 (addon enablement in the lab) → P1 (hub chart) → ongoing. **Depends on:** E0. The +control plane is a single Go binary by design (ADR-0002), which keeps packaging and supply-chain +hardening simple; secrets are referenced from a KMS, never baked into rendered output (E3). + +**Features:** F9.1 hub Helm chart · F9.2 OCM addon / spoke-agent packaging · F9.3 deployment +profiles (light vs heavy) · F9.4 air-gap / on-prem installation · F9.5 upgrade path and addon +version policy. + +### F9.1 — Hub Helm chart + +**What it is.** A Helm chart that installs the Sith hub — the control-plane binary, its +PostgreSQL dependency, configuration, and KMS references for secrets. + +**How it works.** +1. The chart deploys the control-plane binary and wires it to a PostgreSQL instance configured for + RLS (a non-owner app role, F1.5). +2. Secrets are provided as KMS references, not literal values; rendered output never contains a + secret (F3.6). +3. Configuration covers the OCM connection, KMS endpoint, and policy/PDP wiring. +4. The chart supports both profiles (F9.3) via values. + +```mermaid +flowchart TD + A["helm install sith-hub"] --> B["Deploy control-plane binary"] + A --> C["Provision / connect PostgreSQL (non-owner app role, RLS)"] + A --> D["Config: OCM connection, KMS endpoint, PDP wiring"] + A --> E["Secrets as KMS references (never literals in rendered output)"] + B --> F["Hub running, ready to federate"] + C --> F + D --> F + E --> F +``` + +**Acceptance criteria.** +- `helm install` brings up the hub with Postgres (non-owner role, RLS) and correct config. +- No secret literal appears in rendered chart output; secrets are KMS references. + +**Key risk / guardrail.** A chart that rendered secrets into manifests committed to git would leak +them (a predecessor-class failure). Guardrail: secrets are KMS references only, and repo hygiene +(F3.6) keeps rendered output free of sensitive values. + +### F9.2 — OCM addon / spoke-agent packaging + +**What it is.** The Sith spoke agent packaged as an OCM addon so the hub distributes it to spokes +through the OCM addon framework, alongside the pinned `cluster-proxy` and `managed-serviceaccount` +addons. + +**How it works.** +1. The Sith spoke agent (local allowlist + local identity, E4) is packaged as an OCM addon. +2. The hub uses the OCM addon framework to install and manage it on each registered spoke. +3. It is versioned alongside the pinned OCM addons (`cluster-proxy` v0.10.0, + `managed-serviceaccount` v0.10.0). +4. The spoke agent's local allowlist ships with it and is managed per spoke. + +```mermaid +flowchart TD + A["Sith spoke agent packaged as OCM addon"] --> B["Hub: OCM addon framework"] + B --> C["Install on spoke-a"] + B --> D["Install on spoke-b"] + C --> E["Spoke agent: local allowlist + local identity (E4)"] + D --> E + F["Pinned OCM addons: cluster-proxy v0.10.0, managed-serviceaccount v0.10.0"] --> B +``` + +**Acceptance criteria.** +- The spoke agent installs on registered spokes via the OCM addon framework. +- Addon versions are pinned and managed with the OCM addons. + +**Key risk / guardrail.** A spoke agent whose local allowlist could be silently changed from the +hub would weaken the independent second bound. Guardrail: the local allowlist is managed as part +of the spoke's own configuration (defense-in-depth), and addon versions are pinned. + +### F9.3 — Deployment profiles (light vs heavy) + +**What it is.** Two profiles from one chart: a light profile for development/lab (single binary, +minimal dependencies) and a heavy profile for production (HA, external Postgres, cloud KMS). + +**How it works.** +1. The light profile runs the single-binary hub with a minimal Postgres, suitable for `kind`/`k3d` + and demos. +2. The heavy profile runs the hub with high availability, an external managed Postgres, and a + cloud KMS. +3. The same governance and isolation apply in both; the difference is scale and dependency + externalization, not policy. +4. Values select the profile; nothing safety-relevant is disabled in the light profile. + +```mermaid +flowchart TD + A["Chart values: profile?"] --> B["Light: single binary, minimal Postgres (dev/lab)"] + A --> C["Heavy: HA hub, external Postgres, cloud KMS (prod)"] + B --> D["Same governance + isolation"] + C --> D + D --> E["Difference is scale/dependencies, not policy"] +``` + +**Acceptance criteria.** +- Both profiles deploy from one chart; the light profile suits dev/lab and the heavy profile suits + production. +- No safety control is disabled in the light profile. + +**Key risk / guardrail.** A light profile that quietly turned off RLS or KMS to "just work" in dev +would train unsafe habits and mask bugs. Guardrail: safety controls are identical across profiles; +only scale and dependency externalization change. + +### F9.4 — Air-gap / on-prem installation + +**What it is.** Installation in air-gapped or on-prem environments with no outbound internet — +mirrored images and offline addon bundles. + +**How it works.** +1. All images (hub, spoke agent, OCM addons) are mirrored to an internal registry. +2. OCM addon bundles are provided offline so enablement needs no external pulls. +3. The KMS is an on-prem/HSM equivalent reachable within the environment. +4. Install proceeds with no external network dependency. + +```mermaid +flowchart TD + A["Air-gapped environment (no outbound internet)"] --> B["Mirror all images to internal registry"] + A --> C["Provide OCM addon bundles offline"] + A --> D["On-prem KMS / HSM reachable internally"] + B --> E["helm install from internal registry"] + C --> E + D --> E + E --> F["Hub + spokes run with no external pulls"] +``` + +**Acceptance criteria.** +- The hub and spoke agents install and run with no outbound internet, from mirrored images and + offline addon bundles. +- Custody works against an on-prem KMS/HSM. + +**Key risk / guardrail.** A hidden external dependency (an image or addon pulled at runtime) would +break air-gapped installs and could be a supply-chain surprise. Guardrail: all images and addon +bundles are mirrored/offline, and the install is validated with no outbound access. + +### F9.5 — Upgrade path and addon version policy + +**What it is.** A defined upgrade path for the hub and spoke agents, with schema migrations and a +rollback, and an ADR-gated policy for bumping OCM addon versions. + +**How it works.** +1. Hub upgrades run forward schema migrations; a rollback path restores the prior version. +2. Spoke-agent addon upgrades roll out through the OCM addon framework. +3. Bumping a pinned OCM addon version is an ADR-level decision (ADR-0001 update policy), not a + silent change. +4. Upgrades preserve isolation and audit integrity (hash chain continuity, E6). + +```mermaid +flowchart TD + A["Upgrade requested"] --> B["Hub: run forward schema migration"] + B --> C{"Migration healthy?"} + C -- "no" --> D["Roll back to prior version"] + C -- "yes" --> E["Roll out spoke-agent addon upgrade"] + F["OCM addon version bump?"] --> G["ADR-gated decision (not silent)"] + E --> H["Isolation + audit hash-chain preserved"] +``` + +**Acceptance criteria.** +- Hub and spoke upgrades apply with schema migrations and a working rollback. +- OCM addon version bumps are ADR-gated; upgrades preserve isolation and audit continuity. + +**Key risk / guardrail.** A silent addon bump could change security-relevant behavior under the +plan's assumptions. Guardrail: version bumps are ADR-gated and pinned, and upgrades are validated +to preserve isolation and the audit hash chain. + +### E9 exit criteria + +- The hub installs via Helm with RLS-configured Postgres and KMS-referenced secrets; no secret + literal in rendered output. +- The spoke agent ships as an OCM addon distributed through the addon framework, versions pinned. +- Light and heavy profiles deploy from one chart with identical safety controls. +- Air-gapped/on-prem install works with mirrored images, offline bundles, and an on-prem KMS. +- Upgrades apply with migrations and rollback; addon bumps are ADR-gated; audit continuity holds. + +## E10 — Observability and SRE for Sith itself + +**Goal:** make the hub — the crown jewel — observable and operable: metrics, tracing, and +structured logs about Sith's own behavior, SLOs with error budgets, and the hardening a fleet-wide +control plane demands. + +**Phase:** the hardening posture is day-one; the surfaces mature P1 → P3. **Depends on:** E9 +(deployment) and touches every other epic. This epic observes Sith itself — it does not store +other systems' telemetry (that would be a telemetry lake, out of scope). The hub is the +highest-value target and largest blast radius in the estate, so its own operability and hardening +are first-class. + +**Features:** F10.1 metrics · F10.2 distributed tracing · F10.3 structured logging · F10.4 SLOs +and alerting · F10.5 crown-jewel hardening. + +### F10.1 — Metrics + +**What it is.** Metrics about Sith's own health and behavior: control-plane liveness, federation +freshness, intent throughput, refusal and abstention rates, PDP latency. + +**How it works.** +1. The hub exposes metrics for scraping (control-plane health, DB, queue depths). +2. Federation metrics track per-spoke read freshness and dispatch success. +3. Governance metrics track intents proposed/allowed/denied, abstention rate, and approval + latency. +4. These describe Sith itself; Sith does not retain other systems' metric series. + +```mermaid +flowchart TD + A["Sith hub"] --> B["Control-plane metrics: liveness, DB, queues"] + A --> C["Federation metrics: per-spoke freshness, dispatch success"] + A --> D["Governance metrics: intents allowed/denied, abstention rate, approval latency"] + B --> E["Exposed for scraping (about Sith itself)"] + C --> E + D --> E + E --> F["Not a telemetry lake — no other-system series retained"] +``` + +**Acceptance criteria.** +- Sith exposes control-plane, federation, and governance metrics about itself. +- No long-term storage of other systems' metric series (scope guardrail holds). + +**Key risk / guardrail.** Accreting other systems' telemetry would drift Sith into a telemetry +lake. Guardrail: metrics describe Sith's own behavior only; federated health reads stay a bounded +cache (E2), not a series store. + +### F10.2 — Distributed tracing + +**What it is.** Traces that follow an intent's lifecycle across the PEP stages and the hub → spoke +dispatch, so a slow or failed action can be localized. + +**How it works.** +1. Each intent carries a trace/correlation id from proposal through dispatch and outcome. +2. Spans cover the PEP stages (authn → … → dispatch), the PDP call, and per-spoke execution. +3. A trace shows where time went and where a refusal or failure occurred. +4. Traces reference the intent id so they correlate with audit and decision records (E6). + +```mermaid +sequenceDiagram + autonumber + participant C as Client + participant PEP as PEP (spans per stage) + participant PDP as Ardur PDP + participant SP as Spoke agent + C->>PEP: intent (trace id assigned) + PEP->>PEP: span: authn -> role -> verb -> args -> scope + PEP->>PDP: span: PDP query + PDP-->>PEP: verdict + PEP->>SP: span: dispatch + spoke execution + SP-->>PEP: outcome + Note over PEP,SP: trace id == intent id -> correlates with audit + ledger (E6) +``` + +**Acceptance criteria.** +- An intent's lifecycle is traceable across PEP stages, the PDP call, and per-spoke execution. +- Traces correlate with audit/decision records by intent id. + +**Key risk / guardrail.** Traces that captured argument values could leak secrets or sensitive +data. Guardrail: spans carry ids and timings, not secret payloads; the sanitizer (F3.6) applies to +trace attributes too. + +### F10.3 — Structured logging + +**What it is.** Structured, sanitized logs with correlation ids and no secret material. + +**How it works.** +1. Logs are structured (machine-parseable) and carry the intent/trace/correlation id. +2. The error sanitizer (F3.6) strips tokens, keys, and sensitive identifiers before emit. +3. Log levels separate routine operation from security-relevant events (refusals, abstentions, + auth failures). +4. Logs complement, but do not replace, the tamper-evident audit-log (E6). + +```mermaid +flowchart TD + A["Event in hub / spoke agent"] --> B["Structured log record + correlation id"] + B --> C["Sanitizer: strip tokens/keys/sensitive IDs (F3.6)"] + C --> D{"Security-relevant? (refusal/abstention/auth-fail)"} + D -- "yes" --> E["Elevated level for alerting"] + D -- "no" --> F["Routine level"] + E --> G["Emit safe structured log"] + F --> G +``` + +**Acceptance criteria.** +- Logs are structured, correlated by id, and free of secret material. +- Security-relevant events are distinguishable for alerting. + +**Key risk / guardrail.** An unsanitized log line can leak a token. Guardrail: sanitization is +centralized on all emit paths (F3.6), and logs are not treated as the authoritative audit trail — +the tamper-evident ledger (E6) is. + +### F10.4 — SLOs and alerting + +**What it is.** Service-level objectives for the surfaces that matter — read freshness, dispatch +success, PDP latency — with error budgets and alerting. + +**How it works.** +1. SLOs are defined for read freshness (how current the fleet model is), dispatch success rate, + and PDP decision latency. +2. Error budgets track burn; sustained burn pages. +3. Alerts fire on security-relevant conditions too (spikes in refusals/abstentions, auth failures, + signer/KMS errors). +4. SLOs are about Sith's own reliability, since a control plane that is down or slow is itself an + operational risk. + +```mermaid +flowchart TD + A["Define SLOs: read freshness, dispatch success, PDP latency"] --> B["Track error budgets"] + B --> C{"Budget burning fast?"} + C -- "yes" --> D["Page on-call"] + C -- "no" --> E["Within budget"] + F["Security signals: refusal/abstention spikes, auth fails, KMS/signer errors"] --> G["Alert"] +``` + +**Acceptance criteria.** +- SLOs exist for read freshness, dispatch success, and PDP latency, with error budgets. +- Alerts fire on both reliability burn and security-relevant conditions. + +**Key risk / guardrail.** A control plane that degrades silently is an operational hazard for the +whole fleet. Guardrail: SLOs with error budgets and security alerting make degradation visible and +actionable rather than silent. + +### F10.5 — Crown-jewel hardening + +**What it is.** The hardening the hub demands as the highest-value target: signer-key protection, +DB isolation, supply-chain integrity (SBOM, image signing), and least-privilege for the hub's own +service identity. + +**How it works.** +1. The signing key and DEKs live in KMS/HSM (E3); access is tightly scoped and audited. +2. The DB enforces RLS with a non-owner app role (E1); the hub's own service identity is + least-privilege. +3. Supply-chain integrity: images are signed and an SBOM is produced; addon versions are pinned + and verified (threat-model S8). +4. The hub is treated as the crown jewel in threat-modeling and hardening reviews, with the + dispatch path and signer as the most protected assets. + +```mermaid +flowchart TD + A["Hub = crown jewel"] --> B["Signer key + DEKs in KMS/HSM, scoped + audited (E3)"] + A --> C["DB RLS, non-owner app role, hub identity least-privilege (E1)"] + A --> D["Supply chain: signed images + SBOM, pinned + verified addons (S8)"] + A --> E["Dispatch path + signer = most-protected assets"] + B --> F["Compromise blast radius bounded (spoke re-validation, closed vocab)"] + C --> F + D --> F + E --> F +``` + +**Acceptance criteria.** +- Signer key and DEKs are KMS/HSM-protected with scoped, audited access. +- Images are signed with an SBOM; addon versions are pinned and verified; the hub identity is + least-privilege. + +**Key risk / guardrail.** The hub is the single most valuable target; its compromise is the +worst-case scenario. Guardrail: defense-in-depth — even full hub control cannot get a shell on a +spoke or bypass spoke-side re-validation, and the closed vocabulary plus per-spoke allowlists bound +the damage (threat-model S1). + +### E10 exit criteria + +- Sith exposes metrics, traces, and structured sanitized logs about its own behavior, correlated + by intent id, with no secret leakage and no other-system series retention. +- SLOs with error budgets cover read freshness, dispatch success, and PDP latency; alerts cover + reliability and security conditions. +- Crown-jewel hardening is in place: KMS/HSM key custody, RLS + least-privilege hub identity, + signed images + SBOM, pinned/verified addons. + +## E11 — Local fleet client (the adoption wedge) + +**Goal:** ship a single-binary, day-0 local tool that renders every kubeconfig context on the +engineer's machine as one searchable fleet — "k9s for your whole fleet" — with no hub, no OCM, +no account, and no telemetry, so Sith earns adoption before it asks for governance. + +**Phase:** Phase L (day 0) · **Depends on:** E2 (the source-abstract fleet model; local mode is +E2 with a kubeconfig source). This epic is the on-ramp: it needs no OCM and does not gate on +Milestone-0. It is the reshape's centre of gravity. + +**Features:** F11.1 kubeconfig auto-detect + client-side fan-out · F11.2 cache-first fleet +render (CLI + TUI) · F11.3 local web "fleet IDE" (`sith ui`) · F11.4 cross-cluster fleet search ++ correlation · F11.5 per-pod table stakes · F11.6 no-account / no-telemetry / keychain custody. + +### F11.1 — Kubeconfig auto-detect and client-side fan-out + +**What it is.** On launch, `sith` discovers every context in the user's kubeconfig(s) and opens +a read connection to each — the local mode's source adapter for the E2 fleet model. + +**How it works.** +1. Resolve kubeconfig(s) from `$KUBECONFIG` / `~/.kube/config` and enumerate contexts. +2. For each context, honor its exec-credential plugin exactly as kubectl does (aws/gcloud/az + helpers run locally); credentials never leave the machine. +3. Start a read (informer/watch) session per reachable context; mark unreachable contexts. +4. Feed each context's facts into the shared fleet model as a `source = local-kubeconfig` cluster. + +```mermaid +flowchart TD + A["sith launches"] --> B["Enumerate kubeconfig contexts"] + B --> C{"Context reachable? (exec plugin runs locally)"} + C -- "yes" --> D["Open informer/watch read session"] + C -- "no" --> E["Mark context unreachable (surface, don't fail)"] + D --> F["Feed facts into fleet model (source = local-kubeconfig)"] +``` + +**Acceptance criteria.** +- All contexts are detected; each reachable one streams reads; unreachable ones are flagged. +- No credential or kubeconfig is copied off the machine. + +**Key risk / guardrail.** A blocking auth prompt or one dead context stalling startup. Guardrail: +per-context sessions are independent and non-blocking; an unreachable context is surfaced, never +fatal. + +### F11.2 — Cache-first fleet render (CLI + TUI) + +**What it is.** A k9s-style terminal view over the aggregated fleet that renders from a local +cache in tens of milliseconds, plus scriptable CLI verbs (`sith get … --all-clusters`). + +**How it works.** +1. Watch streams hydrate a local store; the UI reads the store first, never the API per keystroke. +2. Views (resources, health, contexts) render from cache; deltas reconcile in the background. +3. A command bar (`:`/cmd-K) offers fuzzy navigation across all clusters at once. +4. CLI verbs render the same model for scripting and SSH use. + +```mermaid +flowchart LR + W["Per-context watch streams"] --> S[("Local fleet cache")] + S --> U["TUI view (renders from cache, <100ms)"] + S --> C["CLI verbs (--all-clusters)"] + A["User keystroke / query"] --> U + U -. "async" .-> W +``` + +**Acceptance criteria.** +- Views and the command bar render under ~100 ms from cache; deltas reconcile without spinners. +- CLI verbs return the same aggregated answers as the TUI. + +**Key risk / guardrail.** Per-keystroke API round-trips (the slow-UI failure). Guardrail: the +store is the single render source; the API is only a background sync target. + +### F11.3 — Local web "fleet IDE" (`sith ui`) + +**What it is.** The same fleet model served as a local web UI on `localhost` — the visual +"Lens-but-better" surface — from the same binary's embedded frontend. + +**How it works.** +1. `sith ui` starts a localhost server binding the embedded web frontend to the local fleet model. +2. The frontend is the *same* one E8 serves in hub mode; here it runs single-user, kubeconfig-direct. +3. It offers aggregated multi-cluster views, fleet search/correlation, and per-pod table stakes. +4. It binds to loopback only; no external listener, no account, no telemetry. + +```mermaid +flowchart TD + A["sith ui"] --> B["Localhost server + embedded frontend"] + B --> C["Same source-abstract fleet model (local source)"] + B --> D{"Bind scope?"} + D -- "loopback only" --> E["Single-user, no account, no telemetry"] + D -- "external" --> X["Refused — local mode is loopback only"] +``` + +**Acceptance criteria.** +- `sith ui` serves the aggregated fleet view on localhost with no account and no telemetry. +- It reuses the same frontend as the hub console (one codebase, two modes). + +**Key risk / guardrail.** Accidentally exposing local mode on a routable interface. Guardrail: +local mode binds loopback only; serving beyond the machine is a hub-mode decision with authn. + +### F11.4 — Cross-cluster fleet search and correlation (local) + +**What it is.** The wedge's signature capability in local mode: one query across every context +("every cluster where `payments` is Degraded", "which contexts run image X"). + +**How it works.** +1. The query engine (E2's F2.3) evaluates a condition across all local-source clusters at once. +2. Results aggregate into one answer listing matching contexts, with any unreachable/stale context flagged. +3. No per-context manual switching; the operator asks once. + +```mermaid +flowchart TD + A["Query across all contexts"] --> B["Evaluate over local fleet model"] + B --> C["Aggregate matches into one answer"] + C --> D{"Any context stale/unreachable?"} + D -- "yes" --> E["Flag coverage gap in result"] + D -- "no" --> F["Return complete cross-cluster answer"] +``` + +**Acceptance criteria.** +- One query returns a correct answer over ≥ 2 kubeconfig contexts; coverage gaps are flagged. + +**Key risk / guardrail.** A silently dropped unreachable context giving a false-complete answer. +Guardrail: coverage is always surfaced (reuses E2/F2.5 staleness semantics). + +### F11.5 — Per-pod table stakes (logs, exec, port-forward, YAML) + +**What it is.** The commodity single-cluster operations whose *absence* drove the Lens exodus — +present in core so the local tool is complete, but not where Sith tries to out-feature Headlamp. + +**How it works.** +1. Logs, exec, port-forward, and YAML view/edit run as ordinary K8s API calls against the + selected context, with the user's own kubeconfig identity. +2. These are local-mode conveniences; they are **not** governed typed intents and carry no + fleet-action semantics. +3. In hub mode the *same person* acts through the governed path instead — local exec is the + user's own kubectl-equivalent, not a Sith-brokered action. + +```mermaid +flowchart TD + A["Select pod in a context"] --> B{"Action"} + B -- "logs / exec / port-forward / YAML" --> C["Direct K8s API call w/ user's kubeconfig identity"] + C --> D["Local convenience (not a governed intent)"] + B -. "fleet action" .-> E["Governed typed intent path (hub, E4/E5)"] +``` + +**Acceptance criteria.** +- Logs, exec, port-forward, and YAML edit work per context in local mode. +- These paths are clearly local conveniences, distinct from the governed action model. + +**Key risk / guardrail.** Confusing local exec with a governed fleet action. Guardrail: local +per-pod ops use the user's own identity and are never dispatched as typed intents; the closed +vocabulary and no-shell rule still bind every *governed* (hub/agent) path. + +### F11.6 — No-account, no-telemetry, keychain custody + +**What it is.** The trust promises that win the Lens-refugee audience: no login wall, no +phone-home, and any local secret kept in the OS keychain (not plaintext). + +**How it works.** +1. Local mode requires no account and starts no telemetry; there is nothing to opt out of. +2. Any secret the local tool must persist goes to the OS keychain (osxkeychain / wincred / + secret-service); a missing keychain fails loudly or encrypts at rest — never silent plaintext. +3. Kubeconfig credentials are read in place and never copied or uploaded. + +```mermaid +flowchart TD + A["Local secret to persist?"] --> B{"OS keychain available?"} + B -- "yes" --> C["Store in keychain"] + B -- "no" --> D["Fail loudly or encrypt-at-rest (never silent plaintext)"] + E["Telemetry / account?"] --> F["None — nothing to opt out of"] +``` + +**Acceptance criteria.** +- No account and no network telemetry in local mode; verified with a network check. +- Secrets never land in plaintext; the keychain fallback is fail-loud, not silent. + +**Key risk / guardrail.** A silent plaintext fallback (the gh-CLI mistake). Guardrail: the +fallback is fail-loud or encrypt-at-rest by construction. + +### E11 exit criteria + +- `brew install sith && sith` → all kubeconfig contexts detected → aggregated fleet view with + cross-cluster search in **< 10 minutes**, offline, nothing leaving the machine. +- The TUI/CLI and `sith ui` render the same fleet model; per-pod table stakes work. +- No account, no telemetry; local secrets are keychain-backed with a fail-loud fallback. +- The local source feeds the *same* E2 fleet model the hub uses (one code path above the source). + +## E12 — Connector framework + +**Goal:** generalize the day-1 hand-written tool adapters into one out-of-process, typed, +versioned connector framework — so integrations scale without the in-process, unversioned sprawl +that drowned Backstage. + +**Phase:** fast-follow (P2 → P3) · **Depends on:** E2 (read adapters feed the fleet model), E4 +(typed-action adapters host verbs). Build the day-1 six by hand first; generalize once the shape +is proven — never a premature ecosystem. + +**Features:** F12.1 out-of-process gRPC connector SDK · F12.2 the three connector kinds · F12.3 +versioning + one-canonical-connector policy · F12.4 generalize the day-1 six. + +### F12.1 — Out-of-process gRPC connector SDK + +**What it is.** Connectors run as separate subprocesses speaking a typed gRPC protocol to the +hub, so a crashing connector cannot take the hub down (the Grafana model). + +**How it works.** +1. Each connector is a subprocess the hub launches and supervises over gRPC. +2. Authors code against an SDK that hides the wire protocol; the hub owns and evolves the format. +3. A panic in a connector is isolated; the hub logs it and continues. + +```mermaid +flowchart LR + H["Sith hub"] -- "gRPC" --> C1["Connector A (subprocess)"] + H -- "gRPC" --> C2["Connector B (subprocess)"] + C1 -. "panic" .-> L["Isolated: hub logs, keeps running"] + A["Author"] --> SDK["Connector SDK (protocol hidden)"] --> C1 +``` + +**Acceptance criteria.** +- Connectors run out-of-process over gRPC; a crashing connector does not crash the hub. +- Authors implement against the SDK, not the wire protocol. + +**Key risk / guardrail.** An in-process shortcut for "just one" connector reintroducing the +crash-coupling. Guardrail: all connectors are out-of-process; no in-process host access exists. + +### F12.2 — The three connector kinds (and nothing else) + +**What it is.** Every connector is exactly one of three kinds; nothing gets arbitrary host access. + +**How it works.** +1. **Read adapter** — pulls normalized facts into the fleet model (e.g. Prometheus, Loki, Helm). +2. **Brokered read-through** — deep-links to the tool's own UI/API; never re-skins it (e.g. Grafana). +3. **Typed-action adapter** — maps a closed verb to the tool's API (e.g. `argocd.sync`). +4. A connector declares its kind; the framework refuses anything outside these three. + +```mermaid +flowchart TD + A["New connector"] --> B{"Declared kind?"} + B -- "read adapter" --> R["Pull normalized facts -> fleet model"] + B -- "brokered read-through" --> D["Deep-link to tool's own UI (no re-skin)"] + B -- "typed-action adapter" --> T["Map a closed verb -> tool API"] + B -- "anything else" --> X["Refused (no arbitrary host access)"] +``` + +**Acceptance criteria.** +- Every connector is one of the three kinds; an out-of-taxonomy connector is rejected. +- Brokered read-through deep-links only; it never re-implements a tool's UI. + +**Key risk / guardrail.** Scope creep into re-skinning (the devops-portal iframe trap). +Guardrail: the taxonomy is closed; "re-skin a tool" is not an expressible connector kind. + +### F12.3 — Versioning and one-canonical-connector policy + +**What it is.** A minor-additive protocol contract and a rule of one canonical connector per +tool — the Terraform discipline that prevents the Backstage redundancy/abandonment failure. + +**How it works.** +1. Major protocol versions delineate compatibility; minor versions are strictly additive. +2. The registry admits **one** canonical connector per target tool, with declared ownership. +3. Breaking a connector's contract is a major-version, reviewed change — never a silent minor bump. + +```mermaid +flowchart TD + A["Connector change"] --> B{"Breaking?"} + B -- "no" --> C["Minor: additive, compatible"] + B -- "yes" --> D["Major: reviewed compatibility break"] + E["New connector for tool T"] --> F{"Canonical connector for T exists?"} + F -- "yes" --> G["Improve the canonical one (no duplicate)"] + F -- "no" --> H["Register as canonical, with owner"] +``` + +**Acceptance criteria.** +- Minor protocol changes are additive; breaks require a major version and review. +- The registry holds one canonical connector per tool with a named owner. + +**Key risk / guardrail.** Overlapping half-maintained connectors (the Backstage marketplace). +Guardrail: one-canonical-per-tool is enforced at registration. + +### F12.4 — Generalize the day-1 six + +**What it is.** Refactor the hand-written Argo CD, Flux, Helm, Prometheus, Loki, and GitHub +adapters onto the framework, proving it against real integrations before opening it wider. + +**How it works.** +1. Reimplement each of the six as a framework connector of its correct kind. +2. Confirm parity with the hand-written behaviour (same facts, same verbs). +3. Only after the six pass does the framework open to further tools (demand-ranked, E-later). + +```mermaid +flowchart LR + A["Hand-written six (Argo/Flux/Helm/Prom/Loki/GitHub)"] --> B["Port each onto the framework"] + B --> C{"Behaviour parity?"} + C -- "yes" --> D["Framework proven -> open to more tools (demand-ranked)"] + C -- "no" --> E["Fix framework before generalizing"] +``` + +**Acceptance criteria.** +- All six run as framework connectors with behaviour parity. +- The framework is opened to new tools only after the six pass. + +**Key risk / guardrail.** Building the framework before proving it (premature abstraction). +Guardrail: the six are the proof; generalization waits on their parity. + +### E12 exit criteria + +- Connectors run out-of-process over a versioned gRPC protocol; a crash is isolated. +- Every connector is one of the three kinds; one canonical connector per tool. +- The day-1 six run on the framework with parity; further tools are demand-ranked, not eager. + +## E13 — Cost read-overlay + +**Goal:** give the fleet a cost dimension by *reading* OpenCost per cluster and rolling it up at +the hub into per-workspace/team views (with GPU columns) — filling the documented OSS fleet-cost +gap without building a metering or optimization engine. + +**Phase:** fast-follow (P3) · **Depends on:** E2 (cost is another fleet-fact kind). This is a +read integration; it never meters, bills, or mutates clusters. + +**Features:** F13.1 OpenCost per-cluster read adapter · F13.2 hub fleet rollup · F13.3 GPU cost +columns · F13.4 freshness + non-goal guard. + +### F13.1 — OpenCost per-cluster read adapter + +**What it is.** A read adapter that pulls per-cluster allocation from an in-cluster OpenCost (or +its metrics) into the fleet model as a `cost` fact kind. + +**How it works.** +1. Where OpenCost runs on a cluster, the adapter reads its allocation output through the E2 read path. +2. Costs are normalized into `cost` fleet facts (per workload/namespace) with source + freshness. +3. Clusters without OpenCost are simply absent from the cost view (surfaced, not faked). + +```mermaid +flowchart TD + A["Cluster with OpenCost"] --> B["Read allocation via E2 read path"] + B --> C["Normalize into cost fleet facts (source + freshness)"] + D["Cluster without OpenCost"] --> E["Absent from cost view (surfaced, not faked)"] +``` + +**Acceptance criteria.** +- Per-cluster costs are ingested as `cost` facts where OpenCost exists; gaps are surfaced. + +**Key risk / guardrail.** Inventing costs for clusters that don't report them. Guardrail: no +OpenCost → no cost fact; the gap is shown, never estimated silently. + +### F13.2 — Hub fleet cost rollup (per-workspace / per-team) + +**What it is.** The capability none of the OSS tools ship free: aggregate per-cluster costs across +the fleet into per-workspace/team rollups at the hub. + +**How it works.** +1. The hub aggregates `cost` facts across all clusters in a workspace. +2. Rollups group by team/label and respect tenant scoping (E1 isolation). +3. Each rollup carries coverage (how many clusters reported) and freshness. + +```mermaid +flowchart TD + A["cost facts across workspace clusters"] --> B["Aggregate at hub (tenant-scoped)"] + B --> C["Group by team/label"] + C --> D["Rollup with coverage + freshness"] +``` + +**Acceptance criteria.** +- A per-workspace/team fleet cost rollup is produced with coverage and freshness stamped. +- Rollups respect tenant isolation. + +**Key risk / guardrail.** A partial rollup read as complete. Guardrail: coverage is always shown. + +### F13.3 — GPU cost columns (DCGM) + +**What it is.** GPU cost/utilization columns in the fleet cost view where DCGM metrics exist — +the MLOps-relevant slice of the cost overlay. + +**How it works.** +1. Where DCGM is present, GPU efficiency/idle-cost facts are ingested alongside CPU/memory cost. +2. The fleet cost view adds GPU columns; MIG/fractional attribution is best-effort where reported. +3. Absent DCGM → no GPU columns for that cluster (surfaced). + +```mermaid +flowchart TD + A["Cluster with DCGM"] --> B["Ingest GPU efficiency/idle-cost facts"] + B --> C["Add GPU columns to fleet cost view"] + D["No DCGM"] --> E["No GPU columns (surfaced)"] +``` + +**Acceptance criteria.** +- GPU cost columns appear where DCGM exists; their absence is surfaced, not faked. + +**Key risk / guardrail.** Over-claiming per-workload GPU precision. Guardrail: attribution is +best-effort and labelled; physical-GPU-level data is not presented as per-pod truth. + +### F13.4 — Freshness and non-goal guard + +**What it is.** The guard that keeps the overlay a *read* — freshness on every cost fact and a +hard line against becoming a metering/optimization engine. + +**How it works.** +1. Every cost fact and rollup carries `observed_at`; stale cost is flagged like any fleet fact. +2. The overlay never writes to clusters, never bills, never auto-rightsizes. +3. Optimization/automation requests are routed to the tools that own them (OpenCost/Kubecost/CAST AI). + +```mermaid +flowchart TD + A["Cost request"] --> B{"Read or mutate?"} + B -- "read/rollup" --> C["Serve with freshness stamp"] + B -- "meter / optimize / rightsize" --> X["Out of scope -> defer to OpenCost/Kubecost/CAST AI"] +``` + +**Acceptance criteria.** +- Cost facts and rollups are freshness-stamped; stale cost is flagged. +- No write/meter/optimize path exists in the overlay. + +**Key risk / guardrail.** Drift into a cost-optimization product. Guardrail: the overlay is +read-only by construction; mutation is not expressible here. + +### E13 exit criteria + +- Per-cluster OpenCost is read into `cost` facts; a per-workspace/team fleet rollup exists with + coverage + freshness; GPU columns appear where DCGM exists. +- The overlay never writes, meters, or optimizes — cost is a read dimension of the fleet model. + +## 4. Roadmap map + +Epics are placed below at their center of gravity — the phase where the bulk of the work lands. +Several span more than one phase: E4 (action federation) ships `gitops.open-pr` in P2 and the +live-mutation verbs in P3; E5 (policy federation) has its seam in P1, the PDP in P2, and the +fan-out reasoning in P3; E6 (audit + ledger) audits reads in P1 and is fully populated in P2; E8 +(console) is a thin read view in P1 and grows the proposal/approval/wave UX in P3; E10 (hardening) +is a day-one posture that matures throughout. A plain line (no arrowhead) between E4 and E5 marks +that they co-develop. + +**Phase L (local mode — E11 + the MCP read tools) ships day-0 and does not gate on Milestone-0.** +It reuses E2's fleet-model code with a **local kubeconfig source** and no OCM, so adoption lands +before the hub exists. E12 (connector framework) and E13 (cost overlay) are fast-follows on top of +E2/E4. Dashed links below mark **shared code**, not a gating dependency. + +```mermaid +flowchart LR + subgraph PL["Phase L — Local mode (day 0, no OCM)"] + E11["E11 Local fleet client"] + E7r["E7 MCP read tools"] + end + subgraph M0["M0 — Falsification"] + E0["E0 OCM substrate & falsification"] + end + subgraph P1["P1 — Read federation (hub)"] + E1["E1 Tenancy & identity"] + E2["E2 Read federation (source-abstract)"] + E8["E8 Operator console"] + E9["E9 Deployment & packaging"] + E10["E10 Observability & SRE"] + end + subgraph P2["P2 — First governed write"] + E3["E3 Credential & key custody"] + E4["E4 Action federation (gitops.open-pr)"] + E5["E5 Policy federation (PEP + Ardur PDP)"] + E6["E6 Audit & decision ledger"] + end + subgraph P3["P3 — Policy federation + MCP write"] + E7["E7 Governed MCP write surface"] + end + subgraph FF["Fast-follow"] + E12["E12 Connector framework"] + E13["E13 Cost read-overlay"] + end + + E11 -. "shares fleet model" .- E2 + E7r -. "same read surface" .- E11 + E11 -. "same frontend" .- E8 + E0 --> E1 + E0 --> E9 + E1 --> E2 + E1 --> E3 + E1 --> E5 + E2 --> E4 + E3 --> E4 + E5 --- E4 + E2 --> E5 + E5 --> E6 + E1 --> E6 + E4 --> E7 + E5 --> E7 + E6 --> E7 + E2 --> E8 + E5 --> E8 + E9 --> E10 + E2 --> E12 + E4 --> E12 + E2 --> E13 +``` + +The falsification gate holds above everything: E1 onward is not funded until E0 returns yes and +ADR-0001 moves to Accepted. Sequencing discipline is never violated: read before write, PR before +mutation, exec never, prod never auto. + +--- + +## 5. Open questions for the owner + +These are decisions that shape the build and are the owner's (GR's) to make. Each is grounded in +a specific epic and is left open on purpose rather than assumed. + +1. **Ardur wiring timing (E5).** ADR-0005 allows a minimal built-in policy to stand in until Ardur + is ready, then be swapped. Do we ship the built-in stand-in for the P2 first write and swap to + Ardur later, or hold the first write until Ardur's PDP / identity-broker / decision-ledger + interfaces are stable enough to wire directly? + +2. **Token issuer and identity source (E1).** What issues the signed tokens whose claims carry + workspace membership and role — an existing OIDC provider, and which one? This fixes the + authn integration and the `memberships[workspace] → role` claim shape. + +3. **KMS/HSM reference target (E3, E9).** What is the reference KMS for the heavy profile and for + air-gapped/on-prem installs (a specific cloud KMS, plus an on-prem/HSM equivalent)? This + determines the envelope-encryption and signing integration and the on-prem story. + +4. **Git host and credential model for `gitops.open-pr` (E4, E3).** Which Git hosts does the first + write target (GitHub, GitLab, Bitbucket), and what is the narrowest credential that can open a + PR on each (a scoped app token, a GitHub App installation, a deploy key)? This is the first + real secret the hub holds. + +5. **Signer key distribution to spokes (E3, E4).** How is the hub's intent-verification public key + distributed to spokes and rotated — through an OCM object, the addon bundle, or another + channel? Spoke-side verification depends on trustworthy key distribution. + +6. **Inter-wave health definition (E5).** What defines "healthy" between waves — Argo CD + application health, an Argo Rollouts analysis run, a custom probe, or a per-workspace choice? + The wave gate is only as good as this signal. + +7. **Default staleness threshold for abstention (E2, E5).** What is the default freshness + threshold that triggers abstention (the plan uses ">10m" as an illustration), and is it + configurable per workspace? This directly tunes how often Sith abstains. + +8. **Spoke-side local allowlist ownership (E4, E9).** Who authors and manages each spoke's local + allowlist, and how is it provisioned and updated? For the second, independent bound to be real + (defense-in-depth), it should not be trivially controllable from the hub alone — confirm the + intended ownership model. + +9. **MCP server exposure and client authentication (E7).** How do external agents authenticate to + the MCP server (the same signed-token model, or per-agent registered identities), and is the + MCP server exposed only within the org boundary or beyond it? This sets the reach of the + governed gateway. + +10. **P1 UI scope (E8).** ADR-0002 allows deferring the UI behind the API and MCP surface. Do we + want any UI at P1, or is a CLI plus the MCP-read surface enough until the P3 approval/wave UX + is needed? + +11. **Tamper-evidence strength (E6).** Is an internal hash-chain sufficient for the target + compliance customers, or do we need external anchoring/notarization (for example a + transparency-log-style external witness) for the audit-log and decision-ledger? + +12. **Local-mode hero surface (E11).** Ship the k9s-style **TUI** first, the local web **"fleet + IDE"** (`sith ui`) first, or both together? The TUI is the leanest day-0 wow; the web UI is + the "Lens-but-better" surface. Which is the hero the wedge leads with? + +13. **Local→hub upgrade UX (E11, E1).** When a user graduates a kubeconfig-direct cluster to an + OCM minion, what is the migration experience — re-import, run side-by-side, or promote + in-place? This is the seam between the adoption wedge and the governed hub. + +14. **MCP read tools in local mode — auth (E7, E11).** In single-user local mode, how does a + local agent authenticate to `sith serve --mcp` — loopback trust, a short-lived local token, + or an OS-keychain-held secret? This sets the day-0 agent story and the shadow-MCP defense. + +15. **Local-mode telemetry stance (E11).** "No telemetry" is the trust promise. Do we want an + explicit, off-by-default, clearly-disclosed opt-in for anonymous usage counts later, or a + permanent hard no? The Lens backlash argues for a hard no; confirm. + diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..ae4558a --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,271 @@ +# Sith — Roadmap + +**Status:** planning · **Date:** 2026-07-10 · **Revision:** consolidated with the July-2026 market +research (E14 Investigation Brain, integration waves, standards-alignment gates, Phase-L build sequence) + +The roadmap is **falsification-first**: each phase must cheaply *disprove* its key +assumption before the next is funded. The first thing we build is not product code — it is +an experiment designed to delete scope. + +Sequencing discipline (never violated): **local before hub · read before write · PR before +mutation · exec never · prod never auto.** + +**Two tracks.** The *local track* (day-0 adoption wedge) ships a single binary that federates +the user's own kubeconfig contexts — it needs no OCM and does not gate on Milestone-0. The +*hub track* (day-N governance) is what Milestone-0 gates. They share **one** fleet-model engine +and **one** enforcement pipeline; the local track is Phase L below, the hub track is +M0 → P1 → P2 → P3. Lead with adoption (Phase L); layer governance on top of the same engine. + +--- + +## Milestone-0 — the OCM falsification test ⟵ *do this first, before any product code* + +**Assumption under test:** OCM `cluster-proxy` + `managed-serviceaccount` really do +deliver outbound-only, cross-network, reach-cluster-local-services connectivity — so we do +**not** need to build a bespoke tunnel/agent. + +**Goal.** Stand up an OCM hub and **2 local spokes** (`kind` or `k3d`), enable the two +addons, and reach a spoke's **in-cluster Grafana / Argo CD** from the hub through the +reverse tunnel, using a **scoped `managed-serviceaccount` token** (not a cluster-admin +kubeconfig). + +**Steps (lab, not product):** +1. Create hub + spoke-a + spoke-b as local clusters. Keep all scratch on + `/Volumes/EXTENDED` (system disk is small). +2. Bootstrap OCM (`clusteradm init` on hub; join spokes). Verify `ManagedCluster` objects. +3. Enable `cluster-proxy` (v0.10.0) and `managed-serviceaccount` (v0.10.0) addons. +4. Deploy a trivial in-cluster service (or Grafana / Argo CD) on each spoke. +5. From the hub, reach that spoke-local service **through cluster-proxy**, authenticating + with an **MSA-projected scoped token**. +6. Confirm the spoke only ever makes **outbound** connections (no inbound hub→spoke port). + +**Exit criteria (the deciding experiment):** +- ✅ **If reachable in ≤ ~1 day of setup** → the "build the agent/tunnel" scope is + **deleted** from Sith. We adopt OCM and spend the saved time on governance. Proceed to + Phase 1. +- ❌ **If it does not work / needs bespoke transport** → the core premise ([ADR-0001](adr/0001-adopt-ocm-vs-bespoke-tunnel.md)) + is wrong. **Stop.** Re-evaluate before writing any product code. (Cheapest possible + place to fail.) + +**Demo.** A terminal recording: hub curls a spoke-local Grafana/Argo CD endpoint via +cluster-proxy using an MSA token, with `tcpdump`/netstat showing spoke connections are +outbound-only. Write up the result in `docs/adr/0001` as the falsification evidence. + +> Milestone-0 is a **lab experiment**, not a feature. Its only artifact is a documented +> yes/no and a short runbook. No Sith product code is written until it passes. + +--- + +## Phase L — Local mode (the adoption wedge, day 0) ⟵ *ships first / in parallel; needs no OCM* + +**Assumption under test:** engineers will install and keep a single-binary local tool that +renders their whole kubeconfig fleet in one view — the "k9s for your whole fleet" wedge — and +that this is the on-ramp to the governed hub. (What is being falsified here is *adoption*, not +transport.) + +**Goal.** `brew install sith && sith` → every kubeconfig context detected → one aggregated, +searchable fleet view with cross-cluster correlation, in under 10 minutes, offline, with +nothing leaving the machine. + +**In scope.** +- One Go binary: `sith` (CLI + k9s-style TUI) and `sith ui` (local web "fleet IDE" on + `localhost`). No account, no telemetry, no server, no agents. +- The **source-abstract fleet model**, populated in local mode from the user's kubeconfig + contexts via client-side fan-out (informer/watch cache), rendered **cache-first**. +- Cross-cluster correlation query and fleet search across all contexts. +- Per-pod table stakes in core: logs, exec, port-forward, YAML view/edit — commodity K8s calls + present because their absence drove the Lens exodus, not the differentiator. +- **Governed MCP read server** (`sith serve --mcp`): the same fleet as annotated read tools, so + an AI agent inherits the read surface. The shadow-MCP lesson makes this a hard requirement — + the sanctioned path must be easier than `npx kubernetes-mcp-server`. +- A **local advisory Investigation Brain** subset (E14) — deterministic, offline hypotheses for + the day-1 failure modes over the locally-reachable lenses; advisory only (a suggested + command / PR diff the user runs). *"k9s for your whole fleet that also tells you why payments + is down."* + +**Build sequence (locked slices — `docs/BUILD-SEQUENCE.md`).** Phase L is delivered as ordered, +always-green slices, each leaving the binary more useful than the last: + +| Slice | What | Issue(s) | +|---|---|---| +| 0 | Foundation walking-skeleton (`fleet.Source` seam + CI) | #47 | +| 1 | Source-abstract model + local-kubeconfig fan-out | #38, #32 | +| 2 | Cache-first render (CLI + TUI) + cross-cluster search | #33 | +| 3 | Per-pod table stakes (logs/exec/port-forward/YAML) | #35 | +| 4 | Local web fleet IDE (`sith ui`) | #34 | +| 5 | No-account / no-telemetry / keychain custody | #36 | +| 6 | MCP read tools (`sith serve --mcp`) | #37 | +| — | Local advisory Investigation Brain (R1–R6, reachable lenses) | #48 | +| P | Packaging & supply chain (parallel; does not gate 1–6) | #27 | + +**Exit criteria.** +- First run to a populated cross-cluster answer in **< 10 minutes**, offline. +- A correlation query returns a correct answer over **≥ 2 kubeconfig contexts**. +- No account, no telemetry, and no credential leaves the machine (verified by an egress test). +- An MCP client (e.g. Claude Code) calls the read tools and gets the same fleet answers. +- The advisory brain surfaces a cited hypothesis + suggested command for a degraded workload, + and **abstains** (naming the missing lens) when a required lens is unreachable. + +**Demo.** `brew install sith && sith` on a laptop with 3 kubeconfig contexts → one fleet view; +"every context where `payments` is Degraded" answered in one query; then Claude Code queries +the same fleet via the MCP read tools; then the brain explains *why* one is degraded. + +> Phase L needs no OCM and does not wait on Milestone-0. It is the adoption wedge; the hub +> track (M0 → P1 → P2 → P3) adds federation and governance on top of the same engine. + +--- + +## Phase 1 — read-only federation (the first vertical) + +**Goal.** From one governed place, assemble a **normalized fleet model** across the 2 +spokes and answer a **cross-cluster** question that single-cluster tools cannot. This is the +**same fleet-model engine as Phase L**, now sourced from OCM-brokered spokes instead of local +kubeconfigs — the read source is abstracted so hub mode and local mode share one code path. + +**In scope.** +- The read source is **abstracted** (local kubeconfig *or* OCM spoke); Phase L's local path + and this hub path are one implementation of the same fleet model. +- Hub read-federation service: pull inventory + health from both spokes via cluster-proxy + + MSA tokens; normalize into the fleet model; stamp **freshness + source cluster**. +- `Workspace` tenancy + signed-token authn + RBAC spine (reader/operator roles), with the + **DB-level RLS backstop present from day one** ([ADR-0003](adr/0003-tenancy-isolation.md)). +- A cross-cluster correlation query (e.g. "every cluster where deployment X is unhealthy"). +- The **policy-hook seam** at the (future) intent boundary, returning "allow" for reads. + +**Exit criteria.** +- A single query returns a correct, tenant-scoped, cross-cluster answer over **≥ 2 spokes**. +- Per-cluster **staleness is visible** in the result. +- A second workspace **cannot** see the first workspace's clusters — verified at the DB + layer, not just the app layer (attempt an app-layer bypass; RLS blocks it). + +**Demo.** "Show me every cluster where `payments` is Degraded" → one answer spanning both +spokes, with a stale cluster flagged, and a tenant-isolation test showing cross-workspace +access denied by the DB backstop. + +--- + +## Phase 2 — first governed typed-intent write (`gitops.open-pr` end-to-end) + +**Goal.** Prove the **action federation + Ardur PDP** path with the **safest possible +write**: `gitops.open-pr` — a proposal a human merges. No cluster mutation yet. + +**In scope.** +- The intent model + closed-vocabulary allowlist (fail-safe) + per-verb arg schema + ([ADR-0004](adr/0004-typed-intent-action-model.md)). +- The PEP enforcement pipeline (authn → membership → role → verb → args → tenant scope → + **Ardur PDP** → elicited approval → scoped identity → caps → **signed dispatch** → + audit + decision-ledger). +- **Ardur as PDP** returning real decisions ([ADR-0005](adr/0005-ai-mcp-ardur-pdp.md)); the + decision-ledger + audit-log both populated. +- `gitops.open-pr` executes by opening a real PR on a target repo. Git credential held via + **KMS envelope, per-tenant** ([ADR-0006](adr/0006-credential-key-custody.md)). +- Spoke-side (or repo-side) independent re-validation of the signed intent. +- The `gitops.open-pr` verb is also exposed as an **MCP write tool** (elicitation-gated), so an + agent can propose it under the same PEP — the first governed *write* surface for agents. + +**Exit criteria.** +- A `gitops.open-pr` intent flows end-to-end and opens a real PR. +- **Zero** cluster credentials reach the center or any AI/agent at any point. +- Every step is in the audit-log; the allow decision is in Ardur's decision-ledger, bound + to a hash of the resolved args. +- A denied intent (wrong role / prod without approval / unknown verb) is refused and logged. + +**Demo.** An operator (then an MCP client) proposes "open a PR to bump replicas for `web` +in workspace X"; Ardur allows with justification; a PR appears; the full +proposed→approved→executed ledger is shown; the same request from a `reader` is refused. + +--- + +## Phase 3 — policy federation (waves / approvals / abstention) + MCP server + +**Goal.** Fan a single intent out to **N clusters** safely, and expose the whole surface as +a **governed MCP server** so external agents inherit the same governance. + +**In scope.** +- **Wave/canary ordering** with a **gate per wave** and a health check between waves. +- **Environment gates + multi-approver** for `prod`; max-clusters-per-intent ceiling. +- **Partial-failure semantics**: stop-on-failure, auto-rollback of the failed wave, + **idempotency/dedupe** on retry. +- **Federation-specific abstention**: refuse fleet-wide action when the targeted set is + incomplete/stale, with an honest message. +- First live-mutation verbs behind all of the above (`argocd.sync`, `rollout.promote`, + `deployment.scale`) — still **never** `exec` or free-form `apply`. +- **MCP server, full write surface**: the live-mutation verbs exposed as MCP write tools gated + by **Elicitation** (2025-06-18), onto the same PEP. (MCP *read* tools shipped in Phase L; the + `gitops.open-pr` write tool shipped in P2; here the fan-out write verbs reach external agents.) + +**Exit criteria.** +- A wave-ordered intent across ≥ 2 spokes runs dev→canary→rest with a gate per wave; a + forced mid-rollout failure triggers auto-rollback of that wave and stops. +- With one spoke made stale, a fleet-wide intent **abstains** with the correct message. +- An external MCP client (e.g. Claude Code) issues a read and a write and is subject to + **identical** governance (approval elicited, decision-ledgered, audited). + +**Demo.** "Sync `payments` across all staging + prod, canary first." Sith plans the waves; +prod requires a second approver; canary passes, one prod cluster fails → that wave rolls +back and the rest halts; then a stale-cluster run shows abstention; then the same run is +driven from an MCP client with the same gates. + +--- + +## The Investigation Brain (E14) — deterministic root-cause across the phases + +The July-2026 market pass found the entire **AI-SRE / auto-triage wave** (k8sgpt, HolmesGPT, +Robusta, Botkube, Komodor, Cleric) converging on one shape: **LLM-agentic, investigate/advise, +read-only or action-gated**. None ships deterministic rule-based root-cause; none ships governed +*typed* action. Sith's **E14 — Investigation Brain** occupies both openings: a **rule-based, +transparent, abstaining** reasoner over E2's four-lens graph that *proposes, never executes*. + +- **Phase L** — a **local advisory subset** (hypotheses + a suggested command/PR the user runs) + over the locally-reachable lenses. Determinism + offline + explainability are the features the + LLM tools structurally cannot offer air-gapped / China / security-conscious estates. +- **P1** — the full deterministic brain over the **four-lens operational graph** (E2 F2.6/F2.7), + correlated by OpenTelemetry semconv keys; the six canonical rules (R1 bad deploy · R2 OOMKilled + · R3 CrashLoopBackOff · R4 config drift · R5 cert expiry · R6 node pressure) reach a *confident* + verdict once the Wave-1 connector core is present, and **abstain** honestly otherwise. +- **P2 / P3** — the **same** rules render a **governed typed-intent proposal** through the PEP: + advisory in local mode, governed in the hub. One brain, two modes. The AI-SRE tools become + *clients* of this governance, not competitors — their advice becomes a typed `plan` Sith gates. + +## Integration waves (E12) — the connector coverage the brain needs + +Connectors ship in four waves (`docs/specs/E2-readfed-brain-integrations.md` §4), each scored by +verb subset, lenses fed, kind (read-adapter / brokered read-through / typed-action), effort tier, +and mode. **Wave 1 is the daily core and is deliberately the exact coverage the six brain rules +need:** + +- **W1 — daily core:** Kubernetes (the substrate) · GitHub · ArgoCD · Prometheus · Elasticsearch · + AWS. With just this, R1/R2/R4/R5/R6 reach *confident* and R3 reaches *detect*. +- **W2 — desired-state/diff:** Helm · Kustomize · kubectl-diff (readers, **not** action targets in v1). +- **W3 — viz/tracing/clouds:** Grafana (deep-link only) · OTel (semconv key backbone) · OpenShift · + Azure · GCP. +- **W4 — long-tail:** OpenSearch · Splunk · Fluentd/FluentBit (**health-only**) · Istio/Linkerd + (mesh → dependency edges) · Docker. + +Scope discipline holds throughout: read log **sinks** not shippers; Grafana is brokered, never +re-skinned; Helm/Kustomize expose no action verbs; telemetry is query-through, never retained. + +## Standards-alignment gates (cross-cutting) + +Not an epic — acceptance gates woven into the epics above (`standards-alignment` label): + +- **MCP 2026-07-28 RC** — OAuth 2.1 + **RFC 8707 audience-bound tokens**, and **enforce-at-execution + not just discovery** (the CVE-2026-46519 bug class, CVSS 8.8). → E7, E4. Build to the stable + primitives; the RC surface will churn. +- **OpenTelemetry Kubernetes semconv** — the correlation join keys for the four-lens graph. → E2. +- **client-go ExecCredential v1** — kubeconfig exec-plugin auth; cloud tokens never persisted. → E1, E11. +- **SLSA L2 + Sigstore/cosign + SBOM** — from the **first tag**, a day-one release gate. → E9. +- **Kubernetes API conventions** — the fleet model reads as idiomatic Kubernetes. → E2. + +## What is deliberately *not* on this roadmap + +- Broad integration count, UI polish, or "autonomy level" as goals in themselves. +- Any verb beyond the closed vocabulary; any `exec`/free-form apply; any secret/RBAC write. +- Re-implementing OCM transport, a scheduler, a portal, or a telemetry store (see + [`SCOPE.md`](SCOPE.md)). +- **No LLM in the critical path for root-cause** — the Investigation Brain (E14) is deterministic; + an LLM is an optional *client*, never the reasoning engine. +- **No "act from chat" free-`kubectl` surface** — the Botkube anti-pattern the threat model rejects. + +Each phase's design decisions are recorded as ADRs; each phase's falsification result is +appended to the relevant ADR as evidence. diff --git a/docs/SCOPE.md b/docs/SCOPE.md new file mode 100644 index 0000000..1fa7798 --- /dev/null +++ b/docs/SCOPE.md @@ -0,0 +1,82 @@ +# Sith — Scope & Defaults + +**Status:** planning · **Date:** 2026-07-08 + +This document is the anti-drift contract. When a feature request arrives, it is checked +against this document first. "It would be useful" is not sufficient to be in scope; it must +be *part of the wedge* (see [`CHARTER.md`](CHARTER.md) §4). + +--- + +## In scope + +- **Local aggregated fleet client (the adoption on-ramp).** A single binary rendering every + kubeconfig context as one searchable fleet — a k9s-style CLI/TUI plus an optional local web + "fleet IDE" — no account, no telemetry, no hub. Distinguished by fleet aggregation and + cross-cluster correlation, **not** per-pod parity with Headlamp/k9s. +- **Read federation** across many clusters, **source-abstract** (local kubeconfig contexts in + day-0 local mode *or* OCM-brokered spokes in day-N hub mode): normalized fleet inventory, + health, alerts, drift, and image/CVE facts, with **cross-cluster correlation** as a + first-class query. +- **Action federation**: a **closed vocabulary of typed intents** dispatched to spokes, + signed, locally re-validated, executed with local scoped identity. +- **Policy federation**: environment gates, multi-approver flows, wave/canary ordering, + partial-failure/rollback, idempotency, and abstention. +- **Governance spine**: multi-tenant `Workspace` isolation, RBAC, signed-token authn, + audit-log (what-happened) + decision-ledger (why-allowed). +- **Governed MCP server** exposing the above so external agents inherit the same governance. +- **Adoption of OCM** (`cluster-proxy`, `managed-serviceaccount`, `ManagedCluster`) as the + transport/identity substrate. +- **Integration with Ardur** as PDP, identity broker, and decision-ledger. + +## Out of scope (non-goals) + +| Not this | Because that is | Owned by | +|---|---|---| +| A developer portal / IDP / service catalog | a different product category | Backstage, Port, Cortex, OpsLevel | +| A GitOps controller / desired-state reconciler | Sith *opens PRs*; it does not reconcile | Argo CD, Flux | +| A multi-cluster scheduler / workload placement | Sith governs *operations*, not placement | Karmada, OCM Placement | +| A telemetry lake / metrics-logs backend | Sith *reads* health; it does not store series | Prometheus, Grafana, Loki, Datadog | +| A bespoke cross-cluster tunnel / agent transport | commodity, security-sensitive plumbing | OCM `cluster-proxy`, Konnectivity, remotedialer | +| A general policy engine | Sith *uses* one (Ardur) at the intent boundary | Ardur / OPA-class engines | +| *Another* single-cluster console / IDE | Sith builds the **aggregated multi-cluster** local view, not per-cluster parity | Headlamp, k9s, Lens (per-cluster) | +| Re-skinning / proxying another tool's own UI | pass-through of a better tool is negative value; Sith **deep-links**, never iframes | the tool's native UI (Grafana, Argo CD, …) | +| An agent-orchestration framework | Sith **governs** the agents that touch clusters; it does not orchestrate them | LangGraph, LangChain, kagent | +| A cost metering / optimization engine | Sith **reads** cost as a fleet overlay (OpenCost rollup); it does not meter or auto-mutate | OpenCost, Kubecost, CAST AI | + +## Permanently excluded from the action model + +These are **never** added to the verb vocabulary, at any phase: + +- `exec` / arbitrary shell into a pod or node. +- Free-form `kubectl apply` of arbitrary manifests. +- **Secret** creation/mutation/read-through. +- **RBAC** object mutation (Role/ClusterRole/Bindings). + +Rationale and the closed vocabulary are in +[ADR-0004](adr/0004-typed-intent-action-model.md). + +## Defaults (safe by default) + +- **Local-first, no phone-home.** Local mode requires **no account** and sends **no + telemetry**; kubeconfig credentials never leave the machine. A permanent promise, not a + default that can be flipped. +- **Read-only first.** New tenants and new integrations start read-only. +- **`prod` never auto-acts.** Any intent targeting a `prod`-labelled cluster requires + approval; multi-cluster `prod` fan-out requires multi-approver. +- **First write is `gitops.open-pr`.** Direct mutations (`argocd.sync`, etc.) are enabled + per-workspace only after the PR path is proven. +- **Dry-run first** for every verb that supports it; show the plan/diff before execute. +- **Fail-safe.** Unknown verb, unschema'd args, unresolved target, stale fleet view, or + missing approval → **refuse**, never "best effort". +- **Abstain on incomplete visibility.** Fleet-wide actions require a fresh, complete view + of the targeted set, or Sith refuses and explains. +- **Least privilege everywhere.** The AI/agent identity ceiling is strictly below the + human's; the spoke identity is scoped to the verb's needs. +- **Everything audited.** Proposed + approved + dry-run + executed, always — for humans and + agents alike. + +## Scope changes + +Adding a verb, adding an OCM addon dependency, or relaxing a default is an **ADR-level +decision** recorded in [`docs/adr/`](adr/). Scope creep is a design defect, not a feature. diff --git a/docs/SITH-NOTION.md b/docs/SITH-NOTION.md new file mode 100644 index 0000000..6f20184 --- /dev/null +++ b/docs/SITH-NOTION.md @@ -0,0 +1,3270 @@ +# Sith — Consolidated Plan (Notion) + +**Status:** planning · **Date:** 2026-07-09 · **License:** Apache-2.0 + +This is the single, self-contained overview of Sith after the local-first dual-mode reshape. It +merges the research synthesis and the reshaped plan into one page: what Sith is, the wedge, the +form factor, the ruthlessly prioritized roadmap, the carry/discard/net-new reckoning against the +predecessor (`devops-portal`), and then **every epic** with its features, how each works, and a +flowchart per feature — followed by the roadmap map and the open questions for the owner. + +> Source of truth lives in the repo: `docs/CHARTER.md`, `docs/SCOPE.md`, `docs/ARCHITECTURE.md`, +> `docs/ROADMAP.md`, `docs/EPICS.md`, and the research under `docs/research/`. Relative links in +> the epics section below point at those repo files. + +--- + +## 1. What Sith is + +Sith is the tool an engineer reaches for the moment they operate **more than one** Kubernetes +cluster. It has two faces on **one Go binary** and one shared engine: + +- **Day 0 — a local fleet client (the adoption wedge).** `brew install sith && sith` opens a + k9s-style view over every kubeconfig context on the machine — plus an optional local web + "fleet IDE" (`sith ui`) that is Lens-but-better. No server, no OCM, no account, no telemetry; + credentials never leave the laptop. This is how Sith gets installed. +- **Day N — a governed hub (the durable moat).** The *same binary* run as `sith hub` becomes a + multi-tenant control plane: clusters join as outbound OCM minions, writes are typed + approval-gated intents through an external policy decision point (Ardur), and the fleet is a + governed MCP server so Claude Code / Codex / kagent inherit the same governance. + +**The reframing that ties it together:** *you earn the right to govern a fleet by first being the +tool the engineer already uses to see it.* Lead with the read wedge; keep governance as the +reason to stay. + +### The de-facto wedge, honestly sized + +The wedge is **"k9s for your whole fleet"** — cross-cluster read + correlation ("every cluster +where `payments` is Degraded", "which clusters run image X") that no OSS tool ships. k9s users +asked for it twice ([#1006](https://github.com/derailed/k9s/issues/1006), +[#2730](https://github.com/derailed/k9s/issues/2730)) and both were closed *not planned*; the +server-based options (Clusterpedia, Karpor) carry day-N install friction; the one tool that ships +it (Aptakube) is closed paid GUI. The demand signal is a *convenience* gap, not an emergency — so +the read view is the beachhead, and the **moat is governed action + governed agent access**, the +position the evidence finds empty across every incumbent (OCM has no approvals/typed-verbs/audit; +the AI-SRE tools are advise-only or autonomy-first; the MCP gateways do coarse on/off, not +fleet-aware approval-gated action). Full evidence: `docs/research/USE-CASE-AND-SHAPE.md`. + +### Two wedges (why the predecessor died) + +- **Adoption wedge — the local aggregated fleet client.** Won by form and trust: single binary, + ten-minute wow, no account, no telemetry (the 2024 CNCF barriers are *complexity 46%*, + *abandonment 46%*, *thin docs 45%* — not governance). The Lens revolt was about an account wall + and logs/shell removed from the OSS build, not a missing feature. +- **Durable wedge — governed action federation with AI as a client.** The moat and the eventual + reason to self-host and standardize. + +`devops-portal` had **neither** — no on-ramp (helm-install-a-platform cliff, 500s on fresh +install) and no moat (it iframed tools that were already better = negative value). Sith holds both +on one shared engine. + +--- + +## 2. Form factor — one binary, three modes + +| Mode | Command | What it is | Who runs it | Phase | +|---|---|---|---|---| +| **CLI + TUI** | `sith` | k9s-style local fleet view over kubeconfig contexts; `sith get pods -A --all-clusters`, `sith correlate`. Read-only, local, zero config | The individual engineer — fastest day-0 wow | Day 0 (wedge) | +| **Local web "fleet IDE"** | `sith ui` | Same fleet model as a local web UI on `localhost` — Lens-but-better, kubeconfig-direct, no account, no telemetry | The engineer who wants a visual surface | Day 0 (wedge) | +| **Governed MCP server** | `sith serve --mcp` | Same fleet exposed to Claude Code / Codex / Cursor as annotated read tools + typed-intent writes | The AI-native engineer — fastest adoption vector | Day 0 / v1 | +| **Hub (federated)** | `sith hub` | Same UI served multi-user: OCM minions reach NAT'd/VPC'd clusters, `Workspace` isolation + RLS, Ardur PDP, multi-approver prod, shared audit | The platform/SRE team | Day N (moat) | + +Key calls (evidence in `docs/research/market-and-form-factor.md`): a hub is **not** required to +federate reads (Aptakube, kubernetes-mcp-server, Karpor all fan out client-side over the +kubeconfig) — OCM is the **day-N** transport for clusters behind NAT/VPCs, not a day-0 dependency. +Render **cache-first** from a local informer/watch cache (never spinner-first). The local mode's +centre of gravity is the **fleet**, not the pod — it is not another single-cluster console. Ship +the embedded web frontend in the Go binary first; a **Tauri** (not Electron) desktop shell is a +fast-follow. + +--- + +## 3. The ruthlessly prioritized roadmap + +Every named capability sits in exactly one bucket; "not now" is as load-bearing as the wedge. + +**Wedge (build first — reason to exist + on-ramp)** +- Local fleet client — `sith` CLI/TUI (fastest wow) **and** `sith ui` (local web fleet IDE), + kubeconfig-direct, no account/telemetry, cache-first, fleet search, logs/exec/port-forward/YAML. +- Source-agnostic read federation + normalized fleet model + cross-cluster correlation (the + shared engine; source = local kubeconfig at day-0 *or* OCM spoke at day-N). +- Governed **MCP read server** from the same binary (`sith serve --mcp`) — v1, not Phase 3. +- Minions (outbound OCM agents) + multi-auth (kubeconfig / API key / JWT / OIDC / short-lived + cloud IAM). +- Governance spine day one: `Workspace` tenancy, signed-token authn, RBAC, forced RLS backstop, + audit-log + decision-ledger. +- First governed write `gitops.open-pr` via the Ardur PDP; its MCP write tool right behind it. +- No-god-key custody: no central admin kubeconfig; KMS-envelope per-tenant DEKs (hub), OS keychain + (local); cosign-signed releases + SLSA L2 + SBOM from the first tag. + +**Fast-follow (right after the wedge proves out)** +- Policy federation — waves/canary, environment gates, multi-approver for prod, partial-failure / + auto-rollback, abstention; live-mutation verbs (`argocd.sync`, `rollout.promote|abort`, + `deployment.scale|restart`). +- Connector framework — out-of-process gRPC, SDK-first, three fixed kinds — generalizing the day-1 + six (Argo CD, Flux, Helm, Prometheus, Loki, GitHub). +- Cost read-overlay — OpenCost rollup + GPU columns; not a metering engine. +- Multi-cloud enumeration + short-lived token minting incl. ACK/CCE/TKE; OpenShift conformant-API. +- Air-gap / multi-arch / registry-relocatable packaging; Tauri desktop shell. + +**Later (real, but not until the above lands)** +- Long-tail read connectors (Datadog, Splunk, Elastic/OpenSearch/Kibana, Terraform state-drift); + ITSM typed actions (Jira/Zendesk/ServiceNow). +- Sith as an MCP *client*; governing LangChain/LangGraph agents (as MCP clients of Sith — Sith + governs, it does not orchestrate). +- MLOps typed verbs (kill zombie GPU jobs, promote model across serving clusters); OpenShift-specific + views. + +**Explicitly not now (the anti-sprawl contract)** +- Re-skinning / proxying tool UIs · telemetry lake / metrics store · metering / optimization engine + · agent-orchestration framework · developer portal / IDP / catalog / scorecards / DORA · GitOps + reconciler · multi-cluster scheduler · Fluentd/Fluent-bit as data sources · Kustomize/Helm as + action targets in v1 · `exec` / free-form apply / Secret / RBAC mutation (permanently + inexpressible) · running SPIRE (support SPIFFE IDs/mTLS, don't force the platform). + +--- + +## 4. Carry / discard / net-new vs `devops-portal` + +| Carry (the good bones) | Discard (the failure modes) | Net-new (what the research says Sith needs) | +|---|---|---| +| Action/exec broker service-layer (clean 1:1 tool→service map) — **redesigned as the PEP**, closed vocabulary, no shell | Shared central admin kubeconfig / inbound-god-kubeconfig — replaced by outbound OCM minions + scoped MSA tokens + no central admin cred | Local aggregated fleet client (CLI/TUI + web) as the adoption wedge (empty OSS slot) | +| Per-org encrypted credential vault (AES-256-GCM key-ring) — **re-architected** as KMS-envelope per-tenant DEKs | Single god key (`TOKEN_ENCRYPTION_KEY` decrypts every tenant) | Cross-cluster correlation as a first-class query in a local tool | +| RBAC + audit spine — kept, **hardened** with signed-token authz + separate decision-ledger | Dead/inert RLS + `x-user-role` header-trust IDOR — replaced by FORCE RLS (non-owner role) + signed-token-only authz | Typed-intent action model + signed dispatch + per-minion local allowlist (two independent blast-radius bounds) | +| Governed AI/MCP ambition — the one salvageable idea — becomes the **core, done right** (real MCP server in v1, elicitation gates, AI-as-client) | All-heavy monolith (~48k LOC, 92 routes; iframes better tools) — replaced by one narrow Go binary + local surfaces, no re-skinning | Policy federation — waves / multi-approver / abstention (novel, empty) | +| `Workspace`-over-clusters tenancy — kept as the isolation anchor with a **real** RLS backstop (day-N hub) | Feature sprawl (12 providers, ~7/10 pass-through) — replaced by closed vocabulary + three connector kinds + a hard scope gate | Governed MCP server as a Kubernetes-fleet-ops gateway (generic gateways aren't fleet-aware) | +| OCM `cluster-proxy` + `managed-serviceaccount` — **re-scope to day-N server mode** | Broken onboarding (platform-install cliff; 500s on fresh install) — replaced by `brew install && sith`, ten-minute wow | Air-gap / multi-arch / registry-relocatable distribution for China/regulated | +| Typed-intent closed vocabulary, no exec — **carry, strengthen** | Features that never ran (dead write path, mock MCP page, qwen2.5:3b) — replaced by falsification-first | Cost read-overlay with fleet rollup + GPU columns; MLOps verbs later | + +--- + +## 5. The epics (features, how-each-works, and a flowchart per feature) + +The rest of this page is the reshaped epic set: **E0–E13**. Each epic states its goal, phase, and +dependencies; each feature states what it is, how it works (numbered), a flowchart, its acceptance +criteria, and its key risk / guardrail. E11 (local client), E12 (connector framework), and E13 +(cost overlay) are the reshape additions; E2 is now source-abstract and the MCP read tools ship in +Phase L. The **roadmap map** and the **open questions for the owner** close the page. + +--- + +## 6. How to read the epics + +### The shape of each epic + +Every epic has an ID and name, a one-line goal, its roadmap phase, its dependencies, and a +list of features. Features are numbered `F.` (for example `F4.3` is the third +feature of epic E4). Each feature is written the same way, with these labelled parts: + +- **What it is** — one or two plain sentences. +- **How it works** — concrete, numbered steps describing the runtime behaviour, followed by + a Mermaid diagram: a `sequenceDiagram` when the point is who-calls-whom over time, a + `flowchart` when the point is branching logic and gates. +- **Acceptance criteria** — how we know the feature is done and correct. +- **Key risk / guardrail** — the one thing most likely to go wrong, and the control that + stops it. + +Each epic closes with epic-level exit criteria — the bar the whole epic clears before the +phase it belongs to is considered met. + +### Flowchart conventions + +- The hub components appear as `PEP`, `RF` (read federation), `AF` (action federation), + `FM` (fleet model), `MCP`, `AUD` (audit-log). Ardur appears as `PDP`, `IDB` (identity + broker), `LEDG` (decision-ledger). A spoke appears as `SP` (Sith spoke agent) and `SVC` + (cluster-local service such as Argo CD / Rollouts / Grafana). +- A decision node (`{ ... }`) with a `refuse` / `deny` / `abstain` branch is drawn on + almost every write path on purpose. Fail-safe is the default: the un-drawn "happy path" + is never the only path. +- Arrows *from* a spoke are always outbound (the spoke dials the hub). No diagram ever + shows the hub opening an inbound connection into a spoke — that property is load-bearing + and is preserved visually. + +### The closed action-verb vocabulary + +Writes are the only dangerous surface, so the set of things Sith can *do* is small, closed, +and reviewed. Every write in this document is one of these verbs and nothing else: + +| Verb | What it does | Idempotent | First shipped | +|---|---|---|---| +| `gitops.open-pr` | Opens a pull request against a target repo — a proposal a human merges | yes (dedupe by content) | P2 (first write) | +| `argocd.sync` | Triggers an Argo CD application sync to already-committed desired state | yes | P3 | +| `argocd.rollback` | Rolls an Argo CD application back to a previous synced revision | no | P3 | +| `rollout.promote` | Promotes an Argo Rollouts canary/blue-green to the next step | no | P3 | +| `rollout.abort` | Aborts an in-progress rollout and returns to stable | yes | P3 | +| `deployment.scale` | Sets replica count on a Deployment via the scale subresource | yes | P3 | +| `deployment.restart` | Triggers a rolling restart of a Deployment | no | P3 | + +**Permanently excluded, at every phase, by every actor including AI:** `exec` / shell into +a pod or node; free-form `kubectl apply` of arbitrary manifests; Secret +create / mutate / read-through; RBAC object mutation. These are not "not yet" — they are +not expressible in the model. Adding *any* new verb is an ADR-level decision +(`ADR-0004`); it is never a routine change. + +### The guardrails (the anti-drift contract, restated so every epic inherits it) + +1. If a capability ships as a maintained OCM addon or upstream project, adopt it rather + than build it. +2. If a feature belongs to "portal", "GitOps controller", "scheduler", or "telemetry + lake", it is out of scope — full stop. +3. The write path may only grow typed verbs in the reviewed closed vocabulary. `exec` and + free-form `apply` are permanently excluded. +4. Multi-tenant isolation, signed intents, per-spoke local enforcement, and scoped + identity are day-one requirements, not later hardening. +5. Fail-safe, never fail-open: anything not explicitly permitted is refused. Unknown verb, + unschema'd args, unresolved target, stale fleet view, or missing approval → refuse. +6. `prod` never auto-acts. Abstention ("I won't act, and here's why") is a first-class, + logged outcome, not an error. +7. The AI is a client of the governance, never a bypass of it. Its identity ceiling is + strictly below the human's; it never holds a cluster credential; it never gets a shell. + +## 7. Epics + +--- + +## E0 — OCM substrate and falsification + +**Goal:** prove, in a lab, that OCM's `cluster-proxy` + `managed-serviceaccount` deliver +outbound-only, cross-network, reach-cluster-local-services connectivity with scoped tokens — +so the whole "build a transport/agent" scope can be deleted. + +**Phase:** M0 · **Depends on:** nothing · **Nature:** a spike, not product code. The only +artifacts are a documented yes/no verdict and a reproducible runbook. No Sith product code is +written until this passes. + +**Features:** F0.1 hub + spoke lab provisioning · F0.2 OCM addon enablement · F0.3 reach a +spoke-local service through the tunnel · F0.4 scoped token projection · F0.5 outbound-only +verification · F0.6 falsification verdict and runbook. + +This epic exists to be able to fail cheaply. If any step cannot be made to work in about a day, +ADR-0001 moves to Rejected and the premise is re-examined before a line of product code exists. + +### F0.1 — Hub + spoke lab provisioning + +**What it is.** A local OCM environment — one hub cluster and two spokes (`spoke-a`, +`spoke-b`) on `kind` or `k3d` — with both spokes registered and healthy on the hub. + +**How it works.** +1. Create three local clusters. Keep all scratch state on `/Volumes/EXTENDED` (the system disk + is small). +2. Run `clusteradm init` on the hub to install the OCM hub control plane. +3. On each spoke, run the join command the hub emits; this installs the klusterlet and starts + an outbound registration request. +4. Accept each spoke's CSR on the hub (`clusteradm accept`). +5. Confirm two `ManagedCluster` objects report `Available`. +6. Capture every command and version into a gitignored runbook. + +```mermaid +flowchart TD + A["Create hub, spoke-a, spoke-b (kind/k3d on /Volumes/EXTENDED)"] --> B["clusteradm init on hub"] + B --> C["Run join command on each spoke (installs klusterlet)"] + C --> D["Spoke sends outbound registration / CSR to hub"] + D --> E["clusteradm accept on hub"] + E --> F{"Both ManagedCluster objects Available?"} + F -- "yes" --> G["Record commands + versions in runbook"] + F -- "no" --> H["Debug registration — if unworkable, flag ADR-0001 risk"] +``` + +**Acceptance criteria.** +- Two spokes are registered and show `Available` on the hub. +- A gitignored runbook reproduces the setup from scratch, with pinned versions. + +**Key risk / guardrail.** Registration friction (CSR, networking) can eat the time budget. +Guardrail: this is exactly the cheap place to hit it — time-box it and treat difficulty here as +a signal about OCM's operational cost, recorded in the runbook. + +### F0.2 — OCM addon enablement (`cluster-proxy` + `managed-serviceaccount`, pinned v0.10.0) + +**What it is.** The two OCM addons Sith depends on, enabled on the hub and both spokes, at +pinned versions verified for July 2026: `cluster-proxy` v0.10.0 and `managed-serviceaccount` +v0.10.0. + +**How it works.** +1. Enable the `cluster-proxy` addon; the hub runs proxy servers, each spoke runs a proxy agent + that dials out to the hub. +2. Enable the `managed-serviceaccount` addon on both spokes. +3. Wait for both addons to report `Available` on each spoke via their `ManagedClusterAddOn` + status. +4. Pin the versions in the runbook so the experiment and any later environment match. + +```mermaid +flowchart TD + A["Enable cluster-proxy addon (pin v0.10.0)"] --> B["Hub: proxy servers start"] + A --> C["Spokes: proxy agents start and dial hub"] + D["Enable managed-serviceaccount addon (pin v0.10.0)"] --> E["Spokes: MSA controller starts"] + B --> F{"Addon status Available on both spokes?"} + C --> F + E --> F + F -- "yes" --> G["Versions pinned in runbook"] + F -- "no" --> H["Inspect addon logs — do not proceed until healthy"] +``` + +**Acceptance criteria.** +- Both addons report `Available` on `spoke-a` and `spoke-b`. +- Versions are pinned and recorded. + +**Key risk / guardrail.** A version drift or a pre-release addon could behave differently from +the plan's assumptions. Guardrail: versions are pinned and any bump is an ADR-gated decision +(ADR-0001 update policy), not a silent upgrade. + +### F0.3 — Reach a spoke-local service through the `cluster-proxy` tunnel + +**What it is.** The deciding step: from the hub, reach an in-cluster service on a spoke (Grafana +or Argo CD, or a trivial stand-in) through the `cluster-proxy` reverse tunnel. + +**How it works.** +1. Deploy a small in-cluster service on each spoke (a plain HTTP service is enough; Grafana or + Argo CD makes the demo concrete). +2. From the hub, issue a request addressed through the `cluster-proxy` proxy service to the + spoke-local service. +3. The request travels the tunnel the spoke agent dialed; the response returns the same way. +4. Repeat for the second spoke to show it generalizes. +5. Time the whole end-to-end setup from F0.1 to here. + +```mermaid +sequenceDiagram + autonumber + participant HUB as Hub (curl / client) + participant CPS as cluster-proxy servers (hub) + participant AG as cluster-proxy agent (spoke, dialed out) + participant SVC as Spoke-local service (Grafana / Argo CD) + Note over AG,CPS: tunnel was DIALED spoke -> hub earlier (outbound-only) + HUB->>CPS: request addressed to spoke-local service + CPS->>AG: forward over the established reverse tunnel + AG->>SVC: reach service inside the spoke network + SVC-->>AG: response + AG-->>CPS: response over tunnel + CPS-->>HUB: response + Note over HUB: repeat for spoke-b, record total setup time +``` + +**Acceptance criteria.** +- The hub gets a valid response from an in-cluster service on both spokes, over the tunnel. +- The total setup time is measured and recorded. + +**Key risk / guardrail.** If reach requires opening an inbound path to the spoke, the core +premise fails. Guardrail: the reach must use only the tunnel the spoke dialed; any need for +inbound access is a falsification failure, recorded as such. + +### F0.4 — Scoped token projection (`managed-serviceaccount`) + +**What it is.** Authenticating that spoke-local reach with a short-lived, scoped +`managed-serviceaccount` token projected to the hub — never a cluster-admin kubeconfig. + +**How it works.** +1. Create a `ManagedServiceAccount` on each spoke scoped to only what the read demo needs. +2. The MSA addon provisions the ServiceAccount on the spoke and projects its token back to the + hub as a secret, with a chosen audience. +3. The hub uses that projected token to authenticate the F0.3 request to the spoke-local + service. +4. Confirm the token is scoped (not cluster-admin) and rotates. + +```mermaid +sequenceDiagram + autonumber + participant HUB as Hub + participant MSA as managed-serviceaccount addon + participant SP as Spoke + participant SVC as Spoke-local service + HUB->>MSA: request a scoped ManagedServiceAccount on the spoke + MSA->>SP: create scoped ServiceAccount + mint token + SP-->>MSA: token (short-lived, scoped, chosen audience) + MSA-->>HUB: project token to hub as a secret + HUB->>SVC: authenticated reach using the projected scoped token + SVC-->>HUB: response + Note over HUB,SP: no cluster-admin kubeconfig ever leaves the spoke +``` + +**Acceptance criteria.** +- Reach in F0.3 is authenticated with a projected, scoped MSA token. +- No cluster-admin kubeconfig is used anywhere in the flow; the token is scoped and rotatable. + +**Key risk / guardrail.** A too-broad token would quietly reintroduce the god-credential +anti-pattern. Guardrail: the MSA is scoped to the minimum the demo needs, and "no admin +kubeconfig in the center" is verified as an explicit check, not assumed. + +### F0.5 — Outbound-only verification + +**What it is.** Evidence that the spoke only ever makes outbound connections to the hub — the +property that lets spokes live in isolated VPCs or behind NAT. + +**How it works.** +1. With the tunnel established and reach working, inspect the spoke's live connections with + `ss` / `netstat`, and optionally `tcpdump`. +2. Confirm every hub-directed connection is outbound (dialed by the spoke). +3. Confirm no inbound hub → spoke port is required for reach to work. +4. Record the observation as evidence for ADR-0001. + +```mermaid +flowchart TD + A["Tunnel up, reach working"] --> B["Inspect spoke connections (ss / netstat / tcpdump)"] + B --> C{"All hub-directed connections outbound?"} + C -- "yes" --> D{"Any inbound hub -> spoke port required?"} + C -- "no" --> X["Falsification concern: not outbound-only — record and escalate"] + D -- "no" --> E["Record outbound-only evidence in ADR-0001"] + D -- "yes" --> X +``` + +**Acceptance criteria.** +- Documented evidence that spoke → hub traffic is outbound-only and no inbound port is needed. + +**Key risk / guardrail.** A hidden inbound dependency would undermine the isolated-VPC claim. +Guardrail: the check is explicit and adversarial (look for inbound requirements, do not just +confirm the happy path), and a negative result is a real finding. + +### F0.6 — Falsification verdict and runbook + +**What it is.** Turning the experiment into a durable decision: fill ADR-0001's falsification +section, move its status, and commit a reproducible runbook plus a short demo capture. + +**How it works.** +1. Record the result, the setup time, and notes in ADR-0001's "Falsification evidence" section. +2. If reachable within about a day, move ADR-0001 to Accepted and delete the transport-build + scope; proceed to Phase 1. If not, move it to Rejected and stop for re-evaluation. +3. Commit the redacted runbook (no secrets) so the experiment is reproducible. +4. Record a short terminal capture: hub reaching a spoke-local service through `cluster-proxy` + with an MSA token, and the outbound-only evidence. + +```mermaid +flowchart TD + A["Experiment complete (F0.1–F0.5)"] --> B{"Reachable via cluster-proxy + MSA in ~1 day?"} + B -- "yes" --> C["ADR-0001 -> Accepted"] + C --> D["Delete 'build the transport' scope"] + D --> E["Proceed to Phase 1 (E1, E2)"] + B -- "no" --> F["ADR-0001 -> Rejected"] + F --> G["Stop — re-evaluate premise before any product code"] + C --> H["Commit runbook + demo capture as evidence"] + F --> H +``` + +**Acceptance criteria.** +- ADR-0001 carries a real verdict (Accepted or Rejected), a setup time, and a link to the + runbook. +- The go/no-go decision for Phase 1 is explicit and recorded. + +**Key risk / guardrail.** The temptation after a "no" is to build the transport anyway. +Guardrail: a "no" verdict is a hard stop for re-evaluation, by design — the whole point of M0 is +that this is the cheapest place to abandon or pivot the premise. + +### E0 exit criteria + +- OCM hub + 2 spokes stand up, both `Available`; both addons healthy at pinned v0.10.0. +- The hub reaches a spoke-local service through the `cluster-proxy` tunnel using a scoped MSA + token, on both spokes, with no cluster-admin kubeconfig anywhere. +- Spoke → hub traffic is verified outbound-only with no inbound port required. +- ADR-0001 records the verdict and setup time; a redacted runbook and a demo capture exist. +- The transport-build scope is deleted (on "yes"), or work stops for re-evaluation (on "no"). + +## E1 — Tenancy and identity + +**Goal:** make "a workspace over many clusters" the single tenancy anchor, with authorization +from signed token claims (never headers), least-privilege RBAC roles, and a real database-level +row-level-security backstop behind app-layer scoping. + +**Phase:** P1 · **Depends on:** E0 (accepted). Isolation is the product here; a control plane +that can see and act across many tenants' fleets must never leak or act across the tenant +boundary. This epic implements the three-layer defense of ADR-0003 from day one. + +**Features:** F1.1 workspace + membership model · F1.2 signed-token authentication · F1.3 RBAC +role gate · F1.4 application-layer tenant scoping · F1.5 database-level RLS backstop · F1.6 +tenant-isolation test suite. + +### F1.1 — Workspace + membership model + +**What it is.** The tenancy data model. A `Workspace` is the scoped tenancy object; every +cluster, policy, intent, decision, audit entry, and fleet fact belongs to exactly one workspace. +A `Membership` grants a subject a role within a workspace. + +**How it works.** +1. `Workspace` carries a `tenant_key` that is the isolation anchor used by every scoping layer. +2. Clusters are explicitly associated with a workspace (tenancy is a workspace over clusters, + never one deployment per cluster). +3. `Membership` maps a subject to a role in a workspace: `reader`, `operator`, `approver`, or + `admin`. +4. Every workspace-scoped table carries a `workspace_id` foreign key so all three isolation + layers have a column to enforce on. + +```mermaid +erDiagram + WORKSPACE ||--o{ CLUSTER : "scopes" + WORKSPACE ||--o{ MEMBERSHIP : "grants" + WORKSPACE ||--o{ POLICY : "owns" + WORKSPACE ||--o{ INTENT : "issues" + WORKSPACE { + id id PK + string name + string tenant_key "isolation anchor" + } + MEMBERSHIP { + id id PK + id workspace_id FK + string subject + string role "reader|operator|approver|admin" + } + CLUSTER { + id id PK + id workspace_id FK + string ocm_managedcluster_ref + } +``` + +**Acceptance criteria.** +- Every workspace-scoped entity carries a `workspace_id`; a cluster belongs to exactly one + workspace. +- A subject's role in a workspace is expressed only through `Membership`. + +**Key risk / guardrail.** A new workspace-scoped table added later without a `workspace_id` +would be a silent leak path. Guardrail: a CI guard (F1.4) forbids workspace-scoped tables that +are not wired into scoping, so an omission fails the build. + +### F1.2 — Signed-token authentication (no header trust) + +**What it is.** Identity, tenant, and role come only from a cryptographically verified token. +Request headers are never trusted for identity — a direct fix for the predecessor's +header-trust IDOR. + +**How it works.** +1. The gateway verifies the session/token signature before anything else; an invalid or absent + token is rejected. +2. Tenant and role are read from the token's `memberships[workspace] → role` claim. +3. Any inbound `x-*-role` / `x-*-tenant` headers are stripped or ignored; they have no effect on + authorization. +4. The verified claims flow downstream as the only source of who-and-where. + +```mermaid +sequenceDiagram + autonumber + participant C as Client (UI / MCP / CLI) + participant GW as API gateway + participant H as Downstream handlers + C->>GW: request (+ signed token, maybe spoofed x-*-role header) + GW->>GW: verify token signature + alt invalid / absent token + GW-->>C: 401 reject + else valid + GW->>GW: read tenant + role from token claims + GW->>GW: strip / ignore any x-*-role, x-*-tenant headers + GW->>H: pass verified claims (headers have no effect) + end +``` + +**Acceptance criteria.** +- A header-injected role has no effect; identity and tenant come only from the signed token. +- A forged or absent token is rejected. + +**Key risk / guardrail.** A single handler reading a header for authorization would reopen the +IDOR. Guardrail: headers are stripped at the gateway and the isolation test suite (F1.6) asserts +a header-injected role changes nothing. + +### F1.3 — RBAC role gate (least privilege) + +**What it is.** A gate that maps the actor's workspace role to the classes of action they may +take, applied before any read or write proceeds. + +**How it works.** +1. `reader` may run reads and correlation queries only. +2. `operator` may additionally propose intents. +3. `approver` may approve gated intents (and read), but the proposer and approver must be + distinct for multi-approver gates. +4. `admin` manages workspace membership and policy bindings, within the workspace only. +5. The gate is fail-safe: a role that does not explicitly permit an action is refused. + +```mermaid +flowchart TD + A["Verified actor + role (from F1.2)"] --> B{"Role permits this action class?"} + B -- "reader: reads only" --> R{"Is this a read?"} + B -- "operator: reads + propose" --> P{"Read or propose-intent?"} + B -- "approver: reads + approve" --> V{"Read or approve?"} + B -- "admin: + manage members/policy (in-workspace)" --> M["Allow scoped admin action"] + R -- "yes" --> OK["Proceed"] + R -- "no" --> DENY["Refuse (fail-safe)"] + P -- "yes" --> OK + P -- "no" --> DENY + V -- "yes" --> OK + V -- "no" --> DENY +``` + +**Acceptance criteria.** +- Each role can do exactly its permitted action classes and no more. +- An action a role does not permit is refused, not best-effort. + +**Key risk / guardrail.** Role creep (an operator quietly gaining approval power) collapses +separation of duties. Guardrail: proposer and approver identities are checked distinct for +multi-approver gates, and the role gate is fail-safe. + +### F1.4 — Application-layer tenant scoping + +**What it is.** A tenant-aware data access layer that injects the current workspace scope into +every query against a workspace-scoped table and hard-fails on any mismatch — covering all such +models, not a subset. + +**How it works.** +1. Every request runs with a resolved workspace context (from F1.2). +2. The data access layer injects `workspace_id = ` into reads and writes on + workspace-scoped tables. +3. If a query returns or targets a row from another workspace, the layer hard-fails rather than + returning it. +4. A CI guard scans for direct access to workspace-scoped tables that bypasses the scoped layer + and fails the build. + +```mermaid +flowchart TD + A["Query on a workspace-scoped table"] --> B["Tenant-aware DAL injects workspace_id = current"] + B --> C{"Any row outside current workspace?"} + C -- "no" --> D["Return / apply within workspace"] + C -- "yes" --> E["Hard-fail (do not return foreign rows)"] + F["CI guard: scan for un-scoped table access"] --> G{"Direct/un-scoped access found?"} + G -- "yes" --> H["Fail the build"] + G -- "no" --> I["Build passes"] +``` + +**Acceptance criteria.** +- All workspace-scoped models are accessed only through the scoped layer. +- A cross-workspace access attempt hard-fails at the app layer; the CI guard blocks un-scoped + access. + +**Key risk / guardrail.** A forgotten filter on one model was the predecessor's silent-leak bug. +Guardrail: the CI guard makes an un-scoped access a build failure, and F1.5 is the independent +backstop if the app layer is ever wrong anyway. + +### F1.5 — Database-level RLS backstop (non-owner role, FORCE RLS, per-request scope) + +**What it is.** PostgreSQL row-level security that is actually enforced, independent of +application code — the backstop the predecessor advertised but left inert. + +**How it works.** +1. The application connects as a non-owner DB role (table owners bypass RLS, so the app must not + be the owner). +2. Every workspace-scoped table has `ENABLE` and `FORCE ROW LEVEL SECURITY`. +3. At the start of each request's transaction, the current workspace is set with + `set_config('sith.workspace_id', , true)`. +4. Each table's RLS policy checks `workspace_id = current_setting('sith.workspace_id')`, so the + database filters foreign rows even if the app layer is bypassed. + +```mermaid +sequenceDiagram + autonumber + participant APP as App (non-owner DB role) + participant DB as PostgreSQL (FORCE RLS) + APP->>DB: BEGIN transaction + APP->>DB: set_config('sith.workspace_id', W, true) + APP->>DB: SELECT / INSERT on workspace-scoped table + DB->>DB: RLS policy: workspace_id = current_setting('sith.workspace_id') + DB-->>APP: only rows for workspace W (foreign rows filtered by DB) + APP->>DB: COMMIT + Note over APP,DB: owner role would bypass RLS — app deliberately is NOT the owner +``` + +**Acceptance criteria.** +- The app connects as a non-owner role; every workspace-scoped table has `FORCE ROW LEVEL + SECURITY`. +- A query with the app-layer scope deliberately removed still returns only the current + workspace's rows. + +**Key risk / guardrail.** RLS that is enabled but not forced, or an app connecting as owner, +would make the backstop inert again. Guardrail: F1.6 includes a test that removing an RLS policy +makes a DB-layer isolation test fail — proving the backstop is live, not decorative. + +### F1.6 — Tenant-isolation test suite + +**What it is.** The test suite that treats isolation as a primary, first-class property, proving +cross-workspace access is impossible at multiple independent layers. + +**How it works.** +1. A cross-workspace read/write is attempted with the app-layer scope deliberately bypassed; the + DB RLS layer must deny it. +2. A forged or absent token is rejected; a header-injected role is shown to have no effect. +3. `targetSelector` and queries are fuzzed with foreign cluster IDs and must always resolve to + empty within-workspace. +4. A negative control: a removed RLS policy makes the DB-layer test fail, proving the backstop is + real. + +```mermaid +flowchart TD + A["Isolation test suite"] --> B["Case 1: app-bypassed cross-workspace query -> DB RLS denies"] + A --> C["Case 2: forged/absent token rejected, header role has no effect"] + A --> D["Case 3: fuzz targetSelector with foreign cluster IDs -> empty"] + A --> E["Negative control: remove an RLS policy"] + B --> F{"All green?"} + C --> F + D --> F + E --> G{"DB-layer test now FAILS?"} + F -- "yes" --> PASS["Isolation verified"] + G -- "yes" --> PASS + G -- "no" --> FAIL["RLS is not actually enforced — fix before shipping"] +``` + +**Acceptance criteria.** +- All isolation cases pass; a removed RLS policy makes the DB-layer test fail. +- Header-injected roles and forged tokens are proven ineffective. + +**Key risk / guardrail.** A green suite that does not actually exercise the DB layer would give +false confidence. Guardrail: the negative control (remove-a-policy-and-watch-it-fail) is part of +the suite, so "green" means the backstop is genuinely enforcing. + +### E1 exit criteria + +- `Workspace` is the single tenancy anchor; every scoped entity carries `workspace_id`. +- Authorization derives only from signed token claims; header-injected roles have no effect. +- RBAC roles (reader/operator/approver/admin) gate action classes fail-safe. +- App-layer scoping covers all workspace-scoped models with a CI guard; DB-level RLS is forced, + per-request, and connects as a non-owner role. +- The isolation suite is green, including the negative control proving RLS is live. + +## E2 — Read federation + +**Goal:** assemble a tenant-scoped, normalized fleet model from OCM-brokered reads and make +cross-cluster correlation a first-class query — answering fleet-wide questions single-cluster +tools structurally cannot. The read source is **abstracted**: facts come from a **local +kubeconfig context** (day-0 local mode, E11) *or* an **OCM-brokered spoke** (day-N hub mode), +behind one common source interface — so the local client and the hub are one code path above +the source. F2.1 defines both source adapters. + +**Phase:** P1 · **Depends on:** E0 (connectivity), E1 (tenancy). Reads never require the write +path and have their own blast radius and rate limits. Least privilege is by construction: the +hub reads only what spoke agents report and only within the scope of the projected MSA token. + +**Features:** F2.1 spoke read collection and normalization · F2.2 fleet model store with +freshness and source stamping · F2.3 cross-cluster correlation query · F2.4 image/CVE fact +ingestion and fleet-wide CVE search · F2.5 staleness surfacing and abstention inputs. + +### F2.1 — Spoke read collection and normalization + +**What it is.** The read path that pulls inventory (deployments, pods, rollouts) and +health/alerts from each spoke through `cluster-proxy` using a scoped MSA token, and normalizes +it into a common shape. + +**How it works.** +1. For each spoke, the read-federation service authenticates with the spoke's projected MSA + token and reaches the spoke's Kubernetes API / cluster-local services through + `cluster-proxy`. +2. It collects inventory (workload objects and their status) and health/alert signals. +3. It maps heterogeneous source shapes into Sith's normalized fleet-fact model (kind = + inventory | health | alert | drift | cve). +4. Reads are bounded to what the MSA token scope and the spoke report allow — the hub cannot + read arbitrary cluster state just because it is the hub. + +```mermaid +sequenceDiagram + autonumber + participant RF as Read-federation service (hub) + participant MSA as MSA token store (hub) + participant CP as cluster-proxy + participant SVC as Spoke API / local services + RF->>MSA: get scoped token for spoke + MSA-->>RF: short-lived scoped token + RF->>CP: read inventory + health (authenticated, scoped) + CP->>SVC: reach over reverse tunnel + SVC-->>CP: raw inventory / health / alerts + CP-->>RF: results (bounded by token scope) + RF->>RF: normalize into fleet-fact model + Note over RF: repeat per spoke, scope limits what can be read +``` + +**Acceptance criteria.** +- Inventory and health are collected from ≥ 2 spokes over `cluster-proxy` with scoped tokens. +- Heterogeneous sources are normalized into one fleet-fact shape. + +**Key risk / guardrail.** A broad token would let the hub over-read a spoke. Guardrail: reads +are bounded by the MSA token scope and by what the spoke agent chooses to report; the hub holds +no admin path. + +### F2.2 — Fleet model store with freshness and source stamping + +**What it is.** The tenant-scoped, cached, normalized fleet model where every record carries the +time it was observed and the cluster it came from. + +**How it works.** +1. Each normalized fact is written to the fleet model with `observed_at` and `source cluster`. +2. Every fact is scoped to its cluster's workspace, so the model inherits E1's isolation + (app-layer scope + DB RLS). +3. The model is a cache of observed state, refreshed on a cadence — it is not a metrics store and + keeps no long time series. +4. `CLUSTER.last_seen` is maintained so a cluster that stops reporting is detectable. + +```mermaid +flowchart TD + A["Normalized facts (from F2.1)"] --> B["Stamp observed_at + source cluster"] + B --> C["Scope to workspace (app layer + DB RLS)"] + C --> D[("Fleet model cache: inventory/health/alert/drift/cve")] + D --> E["Update CLUSTER.last_seen"] + E --> F{"Cluster stopped reporting?"} + F -- "yes" --> G["last_seen ages -> staleness input (F2.5)"] + F -- "no" --> H["Fresh record available to queries"] +``` + +**Acceptance criteria.** +- The fleet model is populated from ≥ 2 spokes, tenant-scoped, each record stamped with freshness + and source. +- The store holds current observed state only — no long-term series. + +**Key risk / guardrail.** Drifting toward storing metrics history would turn Sith into a +telemetry lake (out of scope). Guardrail: the model is a bounded cache of current facts; +retention is deliberately short and history is not a feature. + +### F2.3 — Cross-cluster correlation query + +**What it is.** The differentiator: a query that spans all clusters in a workspace and answers a +question a single-cluster tool cannot, such as "every cluster where deployment `X` is unhealthy". + +**How it works.** +1. A query names a condition (for example, deployment `X` in a Degraded/unhealthy state). +2. The query engine evaluates it against the workspace's fleet model across every cluster at + once — no per-console fan-out by the operator. +3. Results are aggregated into one answer listing the matching clusters. +4. Any cluster whose data is stale is flagged in the result, so the answer is honest about + coverage. + +```mermaid +flowchart TD + A["Query: every cluster where X is unhealthy"] --> B["Resolve within actor's workspace only"] + B --> C["Evaluate condition across all clusters in fleet model"] + C --> D["Aggregate matches into one answer"] + D --> E{"Any matching/relevant cluster stale?"} + E -- "yes" --> F["Flag stale clusters in the result"] + E -- "no" --> G["Return complete cross-cluster answer"] + F --> G +``` + +**Acceptance criteria.** +- One query returns a correct, tenant-scoped answer spanning ≥ 2 spokes. +- Stale clusters are flagged in the result rather than silently dropped. + +**Key risk / guardrail.** A query that silently omits a stale or dark cluster would give a +false-complete answer. Guardrail: staleness is surfaced in every result (F2.5), and completeness +of coverage is explicit. + +### F2.4 — Image/CVE fact ingestion and fleet-wide CVE search + +**What it is.** Ingestion of image and CVE facts per cluster, enabling a fleet-wide search such +as "which clusters run image X with CVE Y". + +**How it works.** +1. Each spoke reports the images running in its cluster (and, where available, associated CVE + findings) as `cve`/`inventory` fleet facts. +2. The hub normalizes and stores these with the same freshness/source stamping. +3. A CVE-search query filters across the workspace's clusters by image or CVE identifier. +4. Results list the matching clusters and workloads, with staleness flagged. + +```mermaid +sequenceDiagram + autonumber + participant SP as Spoke report + participant RF as Read-federation service + participant FM as Fleet model + participant Q as CVE-search query + SP->>RF: image list (+ CVE findings where available) + RF->>FM: store as cve/inventory facts (freshness + source) + Q->>FM: search image X / CVE Y across workspace clusters + FM-->>Q: matching clusters + workloads (stale flagged) + Note over Q,FM: single-cluster tools cannot answer this fleet-wide +``` + +**Acceptance criteria.** +- A fleet-wide search by image or CVE returns the matching clusters across the workspace. +- Results carry freshness and source; stale clusters are flagged. + +**Key risk / guardrail.** Treating a stale image inventory as authoritative could hide a +vulnerable cluster. Guardrail: CVE results inherit freshness stamping, and a stale cluster is +flagged rather than assumed clean. + +### F2.5 — Staleness surfacing and abstention inputs + +**What it is.** The mechanism that turns per-record freshness into a per-cluster staleness signal +and feeds it to correlation results and, later, to action-side abstention. + +**How it works.** +1. Each cluster's freshness is derived from `last_seen` / `observed_at` versus a configured + threshold. +2. A cluster past the threshold is marked stale; the degree of staleness (for example, ">10m") + is available. +3. Query results carry per-cluster staleness so operators see coverage gaps. +4. The same signal is exported for the policy layer, where an incomplete or stale targeted set + drives abstention (E5). + +```mermaid +flowchart TD + A["Per-record observed_at / CLUSTER.last_seen"] --> B["Compare to freshness threshold"] + B --> C{"Past threshold?"} + C -- "yes" --> D["Mark cluster stale (with age, e.g. >10m)"] + C -- "no" --> E["Cluster fresh"] + D --> F["Surface staleness in read results"] + D --> G["Export as abstention input to policy layer (E5)"] + E --> F +``` + +**Acceptance criteria.** +- Per-cluster staleness is computed and visible in read results. +- The staleness signal is available to the policy layer as an abstention input. + +**Key risk / guardrail.** Hidden staleness is the most dangerous failure of a federated read +model. Guardrail: staleness is never hidden — it is surfaced in results and is a first-class +input to abstention, so a partial view cannot masquerade as a complete one. + +### E2 exit criteria + +- A normalized, tenant-scoped fleet model is populated from ≥ 2 spokes, every record stamped with + freshness and source. +- A single cross-cluster correlation query returns a correct answer over ≥ 2 spokes with stale + clusters flagged. +- Fleet-wide image/CVE search works across the workspace. +- Per-cluster staleness is surfaced in results and exported as an abstention input. +- Reads never touch the write path and are bounded by MSA token scope. + +## E3 — Credential and key custody + +**Goal:** hold as few secrets as possible, and protect the ones that remain with envelope +encryption and per-tenant keys so no single leak decrypts every tenant — and with no god +credential in the center. + +**Phase:** the *principles* are day-one (P1); the concrete custody surfaces land when the first +held secret arrives with the write path (P2 — Git credentials and the intent signing key). +**Depends on:** E1 (workspaces are the key-scoping unit). This epic makes the predecessor's +single-env-key blast radius structurally impossible (ADR-0006). + +**Features:** F3.1 no-central-admin-credential posture · F3.2 KMS envelope encryption with +per-tenant data keys · F3.3 intent signing-key custody · F3.4 key rotation and key-ring · F3.5 +boot-time custody checks · F3.6 secret leak prevention. + +### F3.1 — No-central-admin-credential posture + +**What it is.** The structural stance that the hub stores no per-cluster admin kubeconfigs at +all; reach uses scoped MSA tokens and action uses short-lived brokered identity, verified and +executed locally by the spoke. + +**How it works.** +1. Cluster reach = OCM `managed-serviceaccount` scoped tokens (E0/E2), not stored admin + kubeconfigs. +2. Cluster action = Ardur-brokered, short-lived, per-action identity (E5), re-validated and + executed by the spoke with its own local identity. +3. The hub therefore never holds a credential whose compromise means cluster-admin everywhere. +4. Any secret the hub genuinely must hold (Git credential, signing key, integration token) is a + named, bounded exception handled by F3.2–F3.3. + +```mermaid +flowchart TD + A["What does the hub hold?"] --> B["Cluster-admin kubeconfigs? NO — not stored"] + A --> C["Cluster reach -> scoped MSA token (short-lived)"] + A --> D["Cluster action -> Ardur-brokered per-action identity"] + A --> E["Only named exceptions held: Git cred, signing key, integration token"] + E --> F["Each exception protected by KMS envelope (F3.2) / KMS-HSM (F3.3)"] + B --> G["Center compromise != cluster-admin everywhere"] + C --> G + D --> G +``` + +**Acceptance criteria.** +- No per-cluster admin kubeconfig is stored in the hub. +- Every secret the hub does hold is an enumerated exception with a defined custody control. + +**Key risk / guardrail.** A convenience shortcut ("just store the kubeconfig") would reintroduce +the confused-deputy blast radius. Guardrail: storing cluster-admin credentials centrally is +rejected outright by ADR-0001/ADR-0006; reach and action use scoped, brokered, locally +re-validated identity only. + +### F3.2 — KMS envelope encryption with per-tenant data keys + +**What it is.** Any secret the hub holds is encrypted with a per-workspace data key, which is +itself wrapped by a KMS/HSM master key. There is no single process-wide key. + +**How it works.** +1. Each workspace has its own data key (DEK). To store a secret, the DEK encrypts the plaintext + (AES-256-GCM). +2. The DEK is never stored in the clear — it is wrapped (encrypted) by the KMS master key, which + never leaves the KMS. +3. Stored form is the ciphertext plus the wrapped DEK; to read, the KMS unwraps the DEK, which + then decrypts the secret. +4. Compromising one tenant's DEK exposes only that tenant; the KMS master key is never present + in process memory in the clear. + +```mermaid +sequenceDiagram + autonumber + participant APP as Hub + participant KMS as KMS / HSM (master key never leaves) + Note over APP,KMS: ENCRYPT a secret for workspace W + APP->>APP: DEK_W encrypts plaintext (AES-256-GCM) + APP->>KMS: wrap DEK_W with master key + KMS-->>APP: wrapped DEK_W + APP->>APP: store {ciphertext, wrapped DEK_W} + Note over APP,KMS: DECRYPT later + APP->>KMS: unwrap DEK_W + KMS-->>APP: DEK_W (in-process, transient) + APP->>APP: DEK_W decrypts ciphertext +``` + +**Acceptance criteria.** +- Every hub-held secret is encrypted with a per-workspace DEK wrapped by a KMS master key. +- No single process-wide key exists; one tenant's DEK compromise does not expose others. + +**Key risk / guardrail.** A per-tenant key kept in the DB unwrapped would recreate a single point +of compromise. Guardrail: DEKs are always KMS-wrapped at rest; the master key never leaves the +KMS, so the key store alone is not sufficient to decrypt anything. + +### F3.3 — Intent signing-key custody + +**What it is.** The key the hub uses to sign every dispatched intent (E4), held in a KMS/HSM, +treated as the single most sensitive secret in the system. + +**How it works.** +1. The signing key lives in a KMS/HSM; signing is a KMS operation, so the private key never + enters application memory in the clear. +2. Each intent is signed by calling the KMS to produce a signature; each spoke verifies it. +3. The key is rotatable (F3.4); spoke-side local allowlists are the compensating control if the + key is ever compromised. +4. Access to the signing operation is tightly scoped and audited. + +```mermaid +flowchart TD + A["Hub prepares intent to dispatch"] --> B["Request signature from KMS/HSM"] + B --> C["KMS signs (private key never leaves KMS)"] + C --> D["Signed intent dispatched to spoke (E4)"] + D --> E["Spoke verifies signature before acting"] + F["If signing key compromised"] --> G["Spoke local allowlist bounds damage to already-permitted verbs/targets"] +``` + +**Acceptance criteria.** +- Intents are signed via a KMS/HSM operation; the private key never leaves the KMS. +- The signing key is rotatable and its use is audited. + +**Key risk / guardrail.** The signing key is the highest-value secret — its leak lets an attacker +forge intents. Guardrail: it lives in KMS/HSM, rotates, and the spoke's independent local +allowlist (E4) bounds the damage of any forged intent to already-permitted verbs and targets. + +### F3.4 — Key rotation and key-ring + +**What it is.** First-class rotation for data keys and the signing key, with a key-ring that can +decrypt with an old key while encrypting with a new one during rotation. + +**How it works.** +1. Keys carry versions; a key-ring holds the current and recent versions. +2. On a schedule or on demand, a new key version is created and becomes the encrypt/sign key. +3. Existing ciphertext/signatures are still verifiable/decryptable with the older version until + re-wrapped/re-signed. +4. Rotation is a primary test target, since a rotation bug can lock out data or break signature + verification. + +```mermaid +flowchart TD + A["Rotation trigger (schedule or on-demand)"] --> B["Create new key version v(n+1)"] + B --> C["Encrypt / sign new data with v(n+1)"] + B --> D["Keep v(n) in key-ring for decrypt/verify of old data"] + C --> E["Background re-wrap / re-sign old data to v(n+1)"] + D --> E + E --> F{"All migrated to v(n+1)?"} + F -- "yes" --> G["Retire v(n)"] + F -- "no" --> D +``` + +**Acceptance criteria.** +- Data keys and the signing key rotate on schedule and on demand without data loss or broken + verification. +- Old versions remain usable for decrypt/verify until migration completes. + +**Key risk / guardrail.** A rotation that drops an old key before re-encryption would strand data +or break verification. Guardrail: the key-ring keeps old versions until migration completes, and +rotation is exercised explicitly in tests. + +### F3.5 — Boot-time custody checks + +**What it is.** Startup checks that refuse to run if key material is missing, weak, a +placeholder, or if the KMS is unreachable. + +**How it works.** +1. On boot, the hub verifies required key material is present and above an entropy floor. +2. It rejects any placeholder or well-known default value (no "changeme" ever accepted). +3. It verifies KMS reachability so envelope operations will work. +4. If any check fails, the process refuses to start rather than running in a degraded, unsafe + state. + +```mermaid +flowchart TD + A["Process start"] --> B{"Key material present?"} + B -- "no" --> X["Refuse to start"] + B -- "yes" --> C{"Above entropy floor and not a placeholder?"} + C -- "no" --> X + C -- "yes" --> D{"KMS reachable?"} + D -- "no" --> X + D -- "yes" --> E["Start normally"] +``` + +**Acceptance criteria.** +- The hub refuses to start on missing, weak, or placeholder key material, or an unreachable KMS. +- A known-weak or placeholder value is rejected. + +**Key risk / guardrail.** Silently starting with a weak key is how the predecessor accepted bad +key material. Guardrail: boot checks are fail-closed — an unsafe custody state prevents startup +rather than degrading quietly. + +### F3.6 — Secret leak prevention + +**What it is.** The controls that keep secrets out of logs, errors, git, and Helm output — this +is a public repository, so nothing sensitive is ever committed. + +**How it works.** +1. An error sanitizer strips tokens, keys, and sensitive identifiers before anything is logged or + returned. +2. Secrets are never rendered into log lines or into Helm output that lands in git. +3. `.gitignore` pre-empts common secret files; scratch/lab state stays out of the repo. +4. This applies uniformly across control plane, spoke agent, and tooling. + +```mermaid +flowchart TD + A["Log line / error / rendered output"] --> B["Error sanitizer: strip tokens/keys/sensitive IDs"] + B --> C{"Any secret material remaining?"} + C -- "yes" --> D["Redact before emit"] + C -- "no" --> E["Emit safe output"] + F["Repo hygiene: .gitignore + no secrets in Helm output"] --> G["Public repo: nothing sensitive committed"] +``` + +**Acceptance criteria.** +- Logs, errors, and rendered output never contain secret material. +- No secret files are committed; the public repo stays clean. + +**Key risk / guardrail.** A single unsanitized error path can leak a token (the SSRF-reads-env +class of bug). Guardrail: sanitization is centralized and applied to all emit paths, and repo +hygiene is enforced so a leak cannot reach git. + +### E3 exit criteria + +- The hub stores no cluster-admin kubeconfigs; reach and action use scoped/brokered identity. +- Every hub-held secret is envelope-encrypted with a per-tenant DEK wrapped by a KMS master key. +- The intent signing key lives in KMS/HSM and is rotatable; rotation works via a key-ring with no + data loss. +- Boot-time custody checks fail closed; secrets never reach logs, errors, or git. + +## E4 — Action federation + +**Goal:** make the only writes Sith performs be typed intents from a closed verb vocabulary — +signed by the hub, re-validated independently by each spoke, executed with the spoke's own scoped +identity — with `gitops.open-pr` as the first and safest write and no shell ever. + +**Phase:** P2 (the `gitops.open-pr` path) → P3 (live-mutation verbs behind full policy). +**Depends on:** E1 (identity/tenancy), E2 (target resolution against the fleet model), E3 (Git +credential + signing key custody), and E5 (every intent rides the PEP pipeline — E4 and E5 +co-develop). This epic implements ADR-0004. + +**Features:** F4.1 typed intent model and closed vocabulary · F4.2 per-verb arg schema +validation · F4.3 `gitops.open-pr` (first write) · F4.4 signed intent dispatch · F4.5 per-spoke +local allowlist re-validation and local execution · F4.6 dry-run-first execution · F4.7 +live-mutation verbs. + +### F4.1 — Typed intent model and closed vocabulary (fail-safe allowlist) + +**What it is.** The intent as the unit of the write path — an object with a verb drawn from a +closed vocabulary — enforced by a fail-safe allowlist rather than a fail-open denylist. + +**How it works.** +1. An intent carries `{id, workspace, actor, verb, targetSelector, args, justification, + evidenceRefs, signature}`. +2. `verb` must be in the closed vocabulary and have a registered, schema-validated handler; an + unknown verb is refused, not executed. +3. A CI test asserts every handler that reaches a write path is classified against the closed + vocabulary — a forgotten classification fails the build. +4. Adding any verb is an ADR-level change, never a routine edit. + +```mermaid +flowchart TD + A["Proposed intent {verb, targetSelector, args, ...}"] --> B{"verb in CLOSED_VOCAB?"} + B -- "no" --> R["Refuse (fail-safe)"] + B -- "yes" --> C{"registered, schema-validated handler exists?"} + C -- "no" --> R + C -- "yes" --> D["Enter PEP pipeline (E5)"] + E["CI: every write-path handler classified vs vocabulary?"] --> F{"any unclassified?"} + F -- "yes" --> G["Fail the build (not production)"] + F -- "no" --> H["Build passes"] +``` + +**Acceptance criteria.** +- Only vocabulary verbs with registered handlers can execute; unknown verbs are refused. +- The CI classification test fails the build if any write-path handler is unclassified. + +**Key risk / guardrail.** The predecessor's fail-open "denylist of one" made a forgotten +classification auto-executable. Guardrail: the model is a fail-safe allowlist and the CI test +proves nothing reaches a write path unclassified. + +### F4.2 — Per-verb arg schema validation + +**What it is.** Every verb's arguments are validated against a per-verb JSON schema, and +execution is structured API calls — never string interpolation, because there is no shell. + +**How it works.** +1. Each verb registers a JSON schema for its args (for example, `deployment.scale` requires a + valid replica count and a resolved target). +2. Args that fail the schema are refused before any execution. +3. Valid args are passed to a typed handler that makes structured API calls (Argo CD API, + Rollouts API, the scale subresource, a Git PR). +4. No argument is ever concatenated into a command string; the shell path does not exist. + +```mermaid +flowchart TD + A["Intent args for verb V"] --> B{"Args valid against V's JSON schema?"} + B -- "no" --> R["Refuse"] + B -- "yes" --> C["Typed handler for V"] + C --> D["Structured API call (Argo CD / Rollouts / scale subresource / Git PR)"] + D --> E["No string interpolation, no shell — ever"] +``` + +**Acceptance criteria.** +- Args are validated per-verb; invalid args are refused before execution. +- Execution is structured API calls with no shell and no string interpolation. + +**Key risk / guardrail.** String interpolation into a command was the predecessor's RCE path. +Guardrail: verbs map to typed API calls only; there is no shell to inject into, and schema +validation rejects malformed args up front. + +### F4.3 — `gitops.open-pr` (the first and safest write) + +**What it is.** The first write Sith ships: opening a pull request on a target repo — a proposal +a human merges, requiring zero new standing trust and no cluster mutation. + +**How it works.** +1. An operator (or MCP client) proposes `gitops.open-pr` with a target repo/branch and the change + (for example, bump replicas for `web`). +2. The intent passes the full PEP pipeline (E5) and, on allow, the hub uses a Git credential held + via KMS envelope (E3) with the narrowest scope that can open a PR — no direct-push credential. +3. The change is opened as a PR; nothing is applied to any cluster. +4. A human reviews and merges; the merge (via GitOps) is what eventually changes state, keeping + Sith out of the reconcile loop. + +```mermaid +sequenceDiagram + autonumber + participant C as Client (UI / MCP) + participant PEP as PEP pipeline (E5) + participant V as KMS-envelope Git credential (E3) + participant REPO as Target repo + participant H as Human reviewer + C->>PEP: propose gitops.open-pr {repo, branch, change, justification} + PEP->>PEP: authn -> tenant -> role -> verb -> args -> scope -> PDP -> audit + PEP->>V: get narrow-scope Git credential (open-PR only) + V-->>PEP: short-scoped credential + PEP->>REPO: open pull request (no cluster mutation) + REPO-->>PEP: PR URL + PEP-->>C: PR opened (proposal) + H->>REPO: review + merge (GitOps applies later) + Note over PEP: proposed + executed both audited + decision-ledgered +``` + +**Acceptance criteria.** +- A `gitops.open-pr` intent flows end-to-end and opens a real PR. +- Zero cluster credentials reach the center or any AI at any point; the Git credential is + narrow-scope and KMS-protected. + +**Key risk / guardrail.** A broad Git credential (direct push) would let Sith change state +without human review. Guardrail: the credential can only open a PR, not push to protected +branches; the human merge is the change gate, consistent with GitOps orthodoxy. + +### F4.4 — Signed intent dispatch + +**What it is.** The hub signs every dispatched intent and sends it per target, wave-ordered, down +the same reverse tunnel — so each spoke can verify integrity before acting. + +**How it works.** +1. After the PEP allows an intent, the hub requests a signature from the KMS/HSM signing key + (E3). +2. The signed intent is dispatched to each target spoke over the `cluster-proxy` tunnel. +3. Dispatch is per target and wave-ordered (E5): later waves wait for earlier ones and their + gates. +4. Each spoke receives a signed intent it can independently verify (F4.5). + +```mermaid +sequenceDiagram + autonumber + participant PEP as PEP (hub) + participant KMS as KMS/HSM signing key + participant SPa as Spoke agent A + participant SPb as Spoke agent B + PEP->>KMS: sign intent + KMS-->>PEP: signature + Note over PEP,SPb: dispatch is wave-ordered (E5), per target, over the reverse tunnel + PEP->>SPa: signed intent (wave 1) + SPa-->>PEP: outcome + PEP->>SPb: signed intent (wave 2, after wave-1 gate) + SPb-->>PEP: outcome +``` + +**Acceptance criteria.** +- Every dispatched intent is signed by the hub; dispatch is per target and wave-ordered. +- A spoke receives a verifiable signed intent, not a raw command. + +**Key risk / guardrail.** An unsigned or replayable dispatch could be forged or replayed. +Guardrail: intents are signed (integrity anchor) and carry an id for idempotency/dedupe (E5), so +a spoke rejects an unverifiable or duplicate dispatch. + +### F4.5 — Per-spoke local allowlist re-validation and local execution + +**What it is.** A spoke never blindly executes what the hub sends. It verifies the signature, +re-validates the intent against its own local allowlist, and executes with its own scoped local +identity — the second of two independent blast-radius bounds. + +**How it works.** +1. The spoke agent verifies the intent signature against the hub's public key. +2. It re-validates the verb and target against its own local allowlist and local RBAC — + independent of the hub's vocabulary. +3. If both pass, it executes using its own scoped local identity (not a hub-supplied + credential), doing a dry-run first where applicable (F4.6). +4. It returns a per-cluster outcome to the hub; a failed check is a local refusal. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Hub (dispatch) + participant SP as Sith spoke agent + participant SVC as Cluster-local service + PEP->>SP: signed intent + SP->>SP: verify signature + alt signature invalid + SP-->>PEP: refuse (bad signature) + else valid + SP->>SP: re-validate vs LOCAL allowlist + local RBAC + alt not locally allowed + SP-->>PEP: refuse (local allowlist) + else allowed + SP->>SVC: execute with SP's OWN scoped identity (dry-run first) + SVC-->>SP: result + SP-->>PEP: per-cluster outcome + end + end +``` + +**Acceptance criteria.** +- A spoke executes only after verifying the signature and passing its own local allowlist. +- Execution uses the spoke's own scoped identity, never a hub-held credential. + +**Key risk / guardrail.** If the spoke trusted the hub blindly, a hub compromise would be +"execute anything everywhere". Guardrail: the spoke's independent signature check plus local +allowlist plus local RBAC bound the damage of a forged or compromised dispatch to what that spoke +already permits. + +### F4.6 — Dry-run-first execution + +**What it is.** For every verb that supports it, a dry-run runs first and surfaces the plan/diff; +executing is a separate, explicit step. + +**How it works.** +1. On an allowed intent, the handler performs a dry-run against the target and produces a + plan/diff. +2. The plan/diff is surfaced to the proposer (and to any approval step). +3. Execution proceeds only on a separate explicit action, not automatically from the dry-run. +4. Verbs that cannot dry-run declare that, so the operator knows the plan is not previewable. + +```mermaid +flowchart TD + A["Allowed intent"] --> B{"Verb supports dry-run?"} + B -- "yes" --> C["Dry-run -> produce plan/diff"] + C --> D["Surface plan/diff to proposer / approver"] + D --> E{"Explicit execute step taken?"} + E -- "no" --> F["Stop (no change)"] + E -- "yes" --> G["Execute"] + B -- "no" --> H["Declare non-previewable, require explicit execute"] + H --> E +``` + +**Acceptance criteria.** +- Every dry-run-capable verb previews a plan/diff before any change. +- Execution requires a separate explicit step; a dry-run never auto-executes. + +**Key risk / guardrail.** Auto-executing from a plan removes the human's last look. Guardrail: +execute is always a distinct, explicit step separated from the dry-run. + +### F4.7 — Live-mutation verbs (behind full policy) + +**What it is.** The verbs that change cluster state — `argocd.sync|rollback`, +`rollout.promote|abort`, `deployment.scale|restart` — enabled per workspace only after the PR +path is proven, and only behind the full policy layer. + +**How it works.** +1. Live verbs stay disabled until `gitops.open-pr` is proven end-to-end for a workspace. +2. When enabled, each still passes the complete PEP pipeline, is signed, re-validated locally, and + dry-run-first. +3. Fan-out for these verbs is governed by E5 (env gates, wave ordering, partial-failure/rollback, + abstention). +4. `exec` and free-form `apply` remain permanently excluded — live mutation never means arbitrary + mutation. + +```mermaid +flowchart TD + A["Request a live-mutation verb (argocd.sync, rollout.promote, deployment.scale, ...)"] --> B{"PR path proven for this workspace?"} + B -- "no" --> R["Refuse (not yet enabled)"] + B -- "yes" --> C["Full PEP pipeline (E5) + signed dispatch (F4.4)"] + C --> D["Spoke re-validate + dry-run + execute (F4.5/F4.6)"] + D --> E["Fan-out governed: waves, gates, rollback, abstention (E5)"] + F["exec / free-form apply / secret / RBAC mutation"] --> G["Permanently excluded — not expressible"] +``` + +**Acceptance criteria.** +- Live verbs are enabled per workspace only after the PR path is proven, and always ride the full + policy layer. +- `exec` and free-form `apply` remain impossible at every phase. + +**Key risk / guardrail.** Enabling live mutation broadly and early would expose real blast radius +before governance is proven. Guardrail: PR-first per workspace, full policy on every live verb, +and the permanent exclusions hold regardless of phase. + +### E4 exit criteria + +- The write path accepts only closed-vocabulary verbs with registered, schema-validated handlers; + the CI classification test is green. +- `gitops.open-pr` flows end-to-end and opens a real PR with zero cluster credentials in the + center or the AI. +- Every dispatched intent is signed; each spoke independently verifies and re-validates against + its own allowlist and executes with its own identity. +- Dry-run precedes execution for every capable verb; live-mutation verbs are gated behind the PR + proof and the full policy layer. +- No shell, no free-form apply, no secret/RBAC mutation exists anywhere in the path. + +## E5 — Policy federation and governance + +**Goal:** govern the fan-out of a single intent to N clusters — one ordered enforcement pipeline, +Ardur as the policy decision point and identity broker, environment gates and multi-approver +flows, wave/canary ordering with a gate per wave, partial-failure and auto-rollback, idempotency, +and honest abstention when the fleet view is incomplete. + +**Phase:** P1 (the policy-hook seam and pipeline shape, allowing reads) → P2 (Ardur PDP + identity +broker on the first write) → P3 (the full fan-out reasoning). **Depends on:** E1 (identity), E2 +(fleet model and staleness), and co-develops with E4 (every intent rides this pipeline). This is +the genuinely novel, hard part of Sith and implements ADR-0004/ADR-0005. + +**Features:** F5.1 PEP enforcement pipeline · F5.2 policy-hook seam · F5.3 Ardur PDP integration · +F5.4 Ardur scoped-identity broker · F5.5 environment gates and multi-approver · F5.6 wave/canary +ordering with a gate per wave · F5.7 partial-failure semantics and auto-rollback · F5.8 +federation-specific abstention · F5.9 elicited per-action approval bound to an arg-hash. + +### F5.1 — PEP enforcement pipeline + +**What it is.** The single ordered gate every intent passes — from the UI or the MCP server +alike — with no privileged back-door path. Fail-safe: anything not explicitly permitted is +refused. + +**How it works.** The pipeline runs in this order, and any step can refuse: +1. authn from the signed token (never headers); +2. workspace membership; +3. role gate; +4. closed verb vocabulary; +5. arg schema validation; +6. tenant-scoped target resolution (only within the actor's workspace); +7. Ardur PDP query (fan-out aware); +8. elicited approval (per-action, arg-hash bound); +9. scoped identity mint (ceiling below the human); +10. caps/budgets (max clusters per intent, rate limits); +11. signed, wave-ordered dispatch (spoke re-validates independently); +12. audit + decision ledger, always. + +```mermaid +flowchart TD + A["Intent (UI or MCP — same path)"] --> B["authn: signed token, not headers"] + B --> C["workspace membership"] + C --> D["role gate"] + D --> E["closed verb vocabulary"] + E --> F["arg schema validation"] + F --> G["tenant-scoped target resolution"] + G --> H["Ardur PDP query (fan-out aware)"] + H --> I["elicited approval (arg-hash bound)"] + I --> J["scoped identity mint (ceiling < human)"] + J --> K["caps / budgets"] + K --> L["signed, wave-ordered dispatch"] + L --> M["audit + decision ledger"] + B -. "any step may refuse" .-> R["Refuse (fail-safe) + audit"] + H -. "deny" .-> R + I -. "missing" .-> R +``` + +**Acceptance criteria.** +- Every intent, from UI or MCP, passes the same ordered pipeline; there is no privileged path. +- Any step can refuse; a refusal is fail-safe and audited. + +**Key risk / guardrail.** A back-door path (an agent route that skips a gate) would collapse the +whole model. Guardrail: the MCP server and UI are both thin clients onto this one pipeline; +tests assert no write path bypasses it. + +### F5.2 — Policy-hook seam (built early) + +**What it is.** The policy hook at the `executeIntent` boundary, present from Phase 1 returning +"allow" for reads, shaped so Ardur drops in for Phase 2 writes without re-architecture. + +**How it works.** +1. In P1, reads flow through the PEP and the policy hook, which returns allow, and are audited. +2. The hook's interface is shaped for a real PDP from the start: it can return allow, deny, or + require-approval(s). +3. In P2, the hook is wired to Ardur; the surrounding pipeline does not change. +4. This makes the seam a stable boundary rather than a later rewrite. + +```mermaid +flowchart TD + A["Read (P1) or intent (P2+)"] --> B["PEP reaches executeIntent boundary"] + B --> C["Policy hook"] + C --> D{"Phase?"} + D -- "P1 (reads)" --> E["Return allow, audit"] + D -- "P2+ (writes)" --> F["Delegate to Ardur PDP (F5.3)"] + F --> G["allow / deny / require-approval"] + E --> H["Continue pipeline"] + G --> H +``` + +**Acceptance criteria.** +- In P1, every read flows through the PEP and policy hook and is audited. +- The hook interface supports allow/deny/require-approval and accepts Ardur without pipeline + changes. + +**Key risk / guardrail.** Building the write pipeline first and retrofitting policy later invites +an ungoverned interim. Guardrail: the seam exists from day one so the enforcement shape is fixed +before any write is possible. + +### F5.3 — Ardur PDP integration + +**What it is.** At the `executeIntent` boundary, the PEP asks Ardur, for every intent, whether +this actor may issue this verb on these resolved targets in this workspace right now — and gets +back allow, deny, or require-approval(s), fan-out aware. + +**How it works.** +1. The PEP sends Ardur the actor, verb, resolved targets, and workspace. +2. Ardur evaluates versioned, per-tenant, fan-out-aware policy (env gates, multi-approver, caps). +3. It returns allow / deny / require-approval(s) and records the reasons in its decision-ledger + (E6). +4. The PEP acts on the verdict; a deny is a fail-safe refusal. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Sith PEP + participant PDP as Ardur PDP + participant LEDG as Decision-ledger (E6) + PEP->>PDP: may {actor} run {verb} on {resolved targets} in {workspace} now? + PDP->>PDP: evaluate versioned, per-tenant, fan-out-aware policy + PDP->>LEDG: record reasons (why-allowed / why-denied) + PDP-->>PEP: allow / deny / require-approval(s) + alt deny + PEP->>PEP: refuse (fail-safe) + audit + else allow or require-approval + PEP->>PEP: continue pipeline (approval if required) + end +``` + +**Acceptance criteria.** +- Every intent is adjudicated by Ardur with a real verdict; the reasons land in the + decision-ledger. +- A deny results in a fail-safe refusal. + +**Key risk / guardrail.** Hardcoding "which verbs need approval" in Sith would drift from policy +and be unauditable. Guardrail: the decision is Ardur's versioned per-tenant policy, recorded with +reasons, so the "why" is external, explicit, and reviewable. + +### F5.4 — Ardur scoped-identity broker + +**What it is.** Ardur mints the short-lived, per-action, scoped execution identity for each +allowed action, so the AI/agent never holds a cluster credential and its ceiling is strictly +below the human's. + +**How it works.** +1. On allow, the PEP asks Ardur to mint an execution identity for this specific action. +2. The identity is short-lived and scoped to exactly the verb's needs — and capped below the + human actor's own privileges. +3. This complements OCM `managed-serviceaccount` on the spoke side: brokered identity governs the + action; the spoke still executes with its own local identity (E4). +4. The identity expires after the action; nothing long-lived is issued to an agent. + +```mermaid +sequenceDiagram + autonumber + participant PEP as Sith PEP + participant IDB as Ardur identity broker + participant SP as Spoke agent + PEP->>IDB: mint scoped, short-lived identity for THIS action + IDB-->>PEP: per-action identity (ceiling below human, expires) + PEP->>SP: dispatch (identity governs the action) + SP->>SP: execute with SP's OWN local identity (re-validated) + Note over PEP,SP: agent never holds a cluster credential, identity is per-action and expiring +``` + +**Acceptance criteria.** +- Each allowed action uses a short-lived, per-action identity scoped below the human actor. +- No agent ever holds a standing cluster credential. + +**Key risk / guardrail.** A long-lived or over-scoped brokered identity would leak standing power +to an agent. Guardrail: identities are per-action, expiring, and ceiling-capped below the human; +the spoke additionally executes with its own local identity. + +### F5.5 — Environment gates and multi-approver + +**What it is.** Environment-aware gates: `prod` never auto-executes, prod requires N-person +approval, multi-cluster prod fan-out requires multiple approvers, and a max-clusters-per-intent +ceiling applies. + +**How it works.** +1. The resolved targets' environment labels (from the fleet model) determine the gate. +2. Any prod target forces approval; a single approver is never enough for multi-cluster prod + fan-out. +3. A max-clusters-per-intent ceiling caps blast radius regardless of environment. +4. Approvers must be distinct from the proposer (separation of duties, F1.3). + +```mermaid +flowchart TD + A["Resolved targets + env labels"] --> B{"Any prod target?"} + B -- "no" --> C{"Cluster count <= ceiling?"} + B -- "yes" --> D["Require approval (never auto)"] + D --> E{"Multi-cluster prod fan-out?"} + E -- "yes" --> F["Require multiple distinct approvers"] + E -- "no" --> G["Require one approver (distinct from proposer)"] + C -- "yes" --> H["Proceed (subject to PDP)"] + C -- "no" --> R["Refuse: exceeds max-clusters-per-intent"] + F --> C + G --> C +``` + +**Acceptance criteria.** +- No prod target ever auto-executes; multi-cluster prod requires multiple distinct approvers. +- An intent exceeding the max-clusters ceiling is refused. + +**Key risk / guardrail.** A mislabeled environment could route a prod cluster through a +non-prod gate. Guardrail: gates read env labels from the tenant-scoped fleet model, the ceiling +applies regardless of labels, and prod-without-approval is impossible by construction. + +### F5.6 — Wave/canary ordering with a gate per wave + +**What it is.** Sith plans a fan-out as ordered waves (for example dev → staging → one canary prod +→ health-gate → the rest), where each wave is separately gated and a health check runs between +waves. + +**How it works.** +1. The action-federation service plans the target set into ordered waves. +2. Each wave has its own gate; no wave proceeds without passing its gate. +3. Between waves, a health check evaluates the just-completed wave before the next begins. +4. A failed health check or gate stops progression (and triggers F5.7). + +```mermaid +flowchart TD + A["Plan target set into ordered waves"] --> B["Wave 1: dev"] + B --> C{"Wave gate + health check pass?"} + C -- "no" --> S["Stop (invoke partial-failure handling F5.7)"] + C -- "yes" --> D["Wave 2: staging"] + D --> E{"Wave gate + health check pass?"} + E -- "no" --> S + E -- "yes" --> F["Wave 3: one canary prod"] + F --> G{"Canary healthy?"} + G -- "no" --> S + G -- "yes" --> H["Wave 4: remaining prod"] +``` + +**Acceptance criteria.** +- A fan-out runs as ordered waves; each wave has its own gate and a health check between waves. +- No wave proceeds without passing its gate; a failed gate stops progression. + +**Key risk / guardrail.** Fanning out to everything at once is the largest self-inflicted blast +radius. Guardrail: waves with per-wave gates and inter-wave health checks bound how much can +change before a problem is caught. + +### F5.7 — Partial-failure semantics and auto-rollback + +**What it is.** Stop-on-first-failure with auto-rollback of the failed wave, plus +idempotency/dedupe so a retry cannot double-apply. + +**How it works.** +1. If a target in a wave fails, progression stops — later waves do not run. +2. The failed wave is automatically rolled back (using the paired verb where applicable, for + example `rollout.abort` / `argocd.rollback`). +3. Each intent and target carries an idempotency key; a retry with the same key is deduped rather + than re-applied. +4. The outcome (stopped, rolled back, per-cluster results) is recorded. + +```mermaid +flowchart TD + A["Executing a wave"] --> B{"Any target failed?"} + B -- "no" --> C["Wave succeeded -> next wave (F5.6)"] + B -- "yes" --> D["Stop: do not run later waves"] + D --> E["Auto-rollback the failed wave (paired verb)"] + E --> F["Record per-cluster outcomes"] + G["Retry with same idempotency key"] --> H{"Already applied?"} + H -- "yes" --> I["Dedupe: no double-apply"] + H -- "no" --> A +``` + +**Acceptance criteria.** +- A forced mid-rollout failure stops progression and auto-rolls-back that wave. +- A retry with the same idempotency key does not double-apply. + +**Key risk / guardrail.** A retry after a partial failure could double-apply and compound damage. +Guardrail: idempotency keys dedupe retries, and stop-on-failure plus auto-rollback bound a partial +failure to the failed wave. + +### F5.8 — Federation-specific abstention + +**What it is.** When the targeted fleet set is incomplete or stale, Sith refuses fleet-wide action +and says so honestly — a first-class, logged outcome unique to a federated world. + +**How it works.** +1. Before a fan-out, Sith checks the freshness/coverage of the targeted set (from E2/F2.5). +2. If any targeted cluster is stale beyond threshold or not visible, it abstains rather than + acting on a partial view. +3. It returns an honest message naming the gap (for example, "37/40 clusters visible; 3 stale + >10m — I will not issue a fleet sync until they report"). +4. The abstention is logged as a first-class outcome, not an error. + +```mermaid +flowchart TD + A["Fleet-wide intent over targeted set"] --> B["Check coverage + freshness of targeted set (F2.5)"] + B --> C{"All targeted clusters visible and fresh?"} + C -- "yes" --> D["Proceed to fan-out (F5.6)"] + C -- "no" --> E["Abstain: refuse fleet-wide action"] + E --> F["Return honest message: N/M visible, K stale >threshold"] + F --> G["Log abstention as a first-class outcome"] +``` + +**Acceptance criteria.** +- With one spoke made stale, a fleet-wide intent abstains with a correct, honest message. +- The abstention is logged as a first-class outcome, not an error. + +**Key risk / guardrail.** Acting on a partial view (guessing about dark clusters) is more +dangerous than not acting. Guardrail: abstention is mandatory on incomplete/stale coverage and is +logged, so "I won't act" is visible and defensible rather than silent best-effort. + +### F5.9 — Elicited per-action approval bound to an arg-hash + +**What it is.** A required approval that is per-action, non-reusable, and bound to a hash of the +resolved args — so an actor (human or agent) cannot approve one thing and then swap the +arguments. + +**How it works.** +1. When the PDP requires approval, the PEP computes a hash of the fully resolved args and target + set. +2. The approval request is presented (elicited) bound to that hash. +3. The approver approves the specific hashed action; the approval is single-use. +4. Before dispatch, the PEP re-checks that the args still match the approved hash; any mismatch + refuses. + +```mermaid +sequenceDiagram + autonumber + participant PEP as PEP + participant AP as Approver (human) + PEP->>PEP: compute hash of resolved args + targets + PEP->>AP: elicit approval bound to arg-hash + AP-->>PEP: approval (single-use, for this hash) + PEP->>PEP: before dispatch, re-check args hash == approved hash + alt hash mismatch (args changed) + PEP->>PEP: refuse (approve-then-swap blocked) + else match + PEP->>PEP: dispatch (F4.4) + end +``` + +**Acceptance criteria.** +- Approvals are per-action, single-use, and bound to the resolved-args hash. +- Changing the args after approval invalidates it (approve-then-swap is blocked). + +**Key risk / guardrail.** Approve-then-swap (approve a benign action, then change args) is the +classic agent bypass. Guardrail: the approval is bound to an arg-hash re-checked at dispatch, so a +valid signature and a valid approval are both necessary but neither is sufficient if the args +changed. + +### E5 exit criteria + +- Every intent, UI or MCP, passes one ordered PEP pipeline with no privileged path; the policy + seam exists from P1. +- Ardur returns real allow/deny/require-approval verdicts, records reasons, and mints per-action + scoped identities below the human ceiling. +- Prod never auto-acts; multi-cluster prod needs multiple distinct approvers; a max-clusters + ceiling holds. +- A wave-ordered fan-out runs with a gate per wave and inter-wave health checks; a mid-rollout + failure stops and auto-rolls-back that wave; retries dedupe. +- A stale targeted set produces a correct, logged abstention; approvals are arg-hash-bound and + single-use. + +## E6 — Audit and decision ledger + +**Goal:** keep a complete, tamper-evident record of what happened (Sith's audit-log) and why it +was allowed (Ardur's decision-ledger), together forming one agent-action record for humans and +agents alike. + +**Phase:** P1 (reads are audited through the seam) → P2 (the full ledger is populated when the +first write flows). **Depends on:** E1 (tenancy), E5 (the PEP is where entries are produced). The +two records are deliberately separate and complementary (ARCHITECTURE §5, §8). + +**Features:** F6.1 audit-log (what-happened) · F6.2 decision-ledger (why-allowed) · F6.3 +tamper-evidence and append-only storage · F6.4 unified action record with query/export. + +### F6.1 — Audit-log (what-happened) + +**What it is.** Sith's record of every phase of every action — proposed, approved, dry-run, +executed — for reads and writes, always. + +**How it works.** +1. As an intent moves through the pipeline, each phase writes an audit entry (proposed → approved + → dry-run → executed, plus refused/abstained where they occur). +2. Each entry carries the intent id, phase, workspace, actor, and a what-happened detail. +3. Reads are audited too (through the P1 seam), so the log covers the whole governed surface. +4. Entries are written on the same path as enforcement, so there is no unlogged action. + +```mermaid +flowchart TD + A["Intent / read moves through PEP"] --> B["Phase: proposed -> audit entry"] + B --> C["Phase: approved -> audit entry"] + C --> D["Phase: dry-run -> audit entry"] + D --> E["Phase: executed -> audit entry"] + A --> F["Refused / abstained -> audit entry"] + B --> G[("Audit-log: intent_id, phase, actor, workspace, detail, at")] + C --> G + D --> G + E --> G + F --> G +``` + +**Acceptance criteria.** +- Every phase of every action (and every read) produces an audit entry. +- No action reaches a spoke without a corresponding audit trail. + +**Key risk / guardrail.** An action path that skips logging would create a blind spot in exactly +the highest-risk surface. Guardrail: audit writes are on the enforcement path itself, so an +un-audited action is not reachable. + +### F6.2 — Decision-ledger (why-allowed) + +**What it is.** Ardur's record of why each intent was allowed, denied, or sent to approval — +complementing the audit-log's what-happened with the reasons behind the verdict. + +**How it works.** +1. When Ardur adjudicates an intent (F5.3), it records the verdict and the reasons + (which policy, which gate, which approvers required). +2. The decision is keyed to the same intent id as the audit-log. +3. The allow decision is bound to the hash of the resolved args (F5.9), so the "why" is tied to + the exact action approved. +4. Together, the audit-log and decision-ledger reconstruct both what happened and why it was + permitted. + +```mermaid +sequenceDiagram + autonumber + participant PDP as Ardur PDP (F5.3) + participant LEDG as Decision-ledger + participant AUD as Audit-log (F6.1) + PDP->>LEDG: record verdict + reasons (why-allowed / why-denied), arg-hash bound + PDP-->>AUD: same intent_id links the two records + Note over LEDG,AUD: why-allowed (Ardur) + what-happened (Sith) = complete action record +``` + +**Acceptance criteria.** +- Every verdict records its reasons in the decision-ledger, keyed to the intent id. +- The allow decision is bound to the resolved-args hash. + +**Key risk / guardrail.** A verdict without recorded reasons is unauditable ("it said yes, but +why?"). Guardrail: reasons are recorded with each verdict and bound to the arg-hash, so the "why" +is specific and reviewable. + +### F6.3 — Tamper-evidence and append-only storage + +**What it is.** Both records are append-only and tamper-evident, so an attacker who reaches the +store cannot silently rewrite history to hide an action. + +**How it works.** +1. Entries are append-only — no in-place update or delete on the enforcement path. +2. Each entry is chained to the prior entry's hash, so any later edit breaks the chain. +3. An integrity verification can walk the chain and detect any altered or removed entry. +4. This makes the audit-log and decision-ledger forensic assets even under partial compromise. + +```mermaid +flowchart TD + A["New audit / decision entry"] --> B["Compute hash over entry + previous entry's hash"] + B --> C[("Append-only store (chained hashes)")] + C --> D["Integrity check walks the chain"] + D --> E{"Any entry altered or missing?"} + E -- "yes" --> F["Tampering detected (chain broken)"] + E -- "no" --> G["History intact"] +``` + +**Acceptance criteria.** +- Entries are append-only and hash-chained; an altered or removed entry breaks verification. +- An integrity check can detect tampering after the fact. + +**Key risk / guardrail.** Tampering hides an attack (S-class: forensics destroyed). Guardrail: the +hash chain makes any edit detectable, so even a store-level compromise cannot silently rewrite +what happened. + +### F6.4 — Unified action record with query/export + +**What it is.** The ability to correlate the audit-log and decision-ledger by intent id into one +view, and to export it for compliance and incident review — tenant-scoped. + +**How it works.** +1. A query joins audit entries and decision entries on intent id to show the full lifecycle of an + action. +2. Queries and exports are tenant-scoped (E1), so one workspace never sees another's records. +3. An export produces a portable record (what happened, when, who, why allowed) for compliance + owners. +4. This answers the security/compliance question: prove what operators — human and agent — were + allowed to do and did do across the fleet. + +```mermaid +flowchart TD + A["Query by intent_id (tenant-scoped)"] --> B["Join audit-log (what) + decision-ledger (why)"] + B --> C["Unified lifecycle: proposed -> approved -> executed + reasons"] + C --> D{"Export requested?"} + D -- "yes" --> E["Portable compliance record (workspace-scoped)"] + D -- "no" --> F["Show in console / API"] +``` + +**Acceptance criteria.** +- Audit and decision records correlate by intent id into one lifecycle view. +- Query/export is tenant-scoped and produces a portable compliance record. + +**Key risk / guardrail.** A cross-tenant leak in an export would be a serious confidentiality +breach. Guardrail: query and export inherit E1's tenant scoping and DB RLS, so a record is only +ever visible within its own workspace. + +### E6 exit criteria + +- Every phase of every action and read produces an audit entry; no unlogged action reaches a + spoke. +- Every verdict records its reasons in the decision-ledger, keyed to intent id and bound to the + arg-hash. +- Both records are append-only and hash-chained; tampering is detectable. +- Audit and decision correlate into one tenant-scoped lifecycle view with a portable export. + +## E7 — Governed MCP server + +> **Reshape note:** the MCP **read** tools (F7.1) ship early, in **Phase L** with the local +> client (E11) — the shadow-MCP lesson makes the sanctioned read path a day-0 requirement. The +> **write** tools (F7.2–F7.5) stay gated behind the governed write path (P2 `gitops.open-pr`, +> then P3 fan-out). Everything still rides the same PEP; nothing about the enforcement changes. + +**Goal:** expose the federated read and action surface as an MCP server so external agents +(Claude Code, Codex, kagent) become clients that inherit exactly the governance a human has — the +same PEP, the same PDP, the same closed vocabulary, the same audit — a governed MCP gateway to the +whole fleet. + +**Phase:** P3 · **Depends on:** E2 (reads), E4 (write vocabulary), E5 (PEP/PDP), E6 (audit + +ledger). MCP tool annotations shipped in the 2025-03-26 spec and are hints, not guarantees, so +enforcement is server-side; Elicitation shipped in the 2025-06-18 spec as the native primitive +for human-in-the-loop approval. Implements ADR-0005. + +**Features:** F7.1 MCP read tools · F7.2 MCP write tools mapped 1:1 to the closed vocabulary · +F7.3 elicitation-based approval on writes · F7.4 external agent as governed client · F7.5 AI +safety rules. + +### F7.1 — MCP read tools + +**What it is.** Read tools (`fleet.inventory`, `fleet.health`, `fleet.correlate`, +`fleet.cve-search`) carrying `readOnlyHint: true`, hitting the fleet model, scoped to the caller's +workspace. + +**How it works.** +1. An external agent calls a read tool over MCP. +2. The MCP layer is a thin client onto the same PEP; the read is tenant-scoped to the caller's + workspace. +3. The fleet model (E2) answers, including cross-cluster correlation and CVE search. +4. The read is audited like any other (E6); no gate beyond tenant scope is needed for reads. + +```mermaid +sequenceDiagram + autonumber + participant AG as External agent (MCP client) + participant MCP as Sith MCP server + participant PEP as PEP (tenant scope) + participant FM as Fleet model (E2) + AG->>MCP: call fleet.correlate (readOnlyHint: true) + MCP->>PEP: same path as UI — resolve workspace scope + PEP->>FM: query within caller's workspace + FM-->>PEP: cross-cluster answer (stale flagged) + PEP-->>MCP: result (audited) + MCP-->>AG: result +``` + +**Acceptance criteria.** +- Read tools return workspace-scoped fleet answers, including correlation and CVE search. +- Reads are audited and carry `readOnlyHint`. + +**Key risk / guardrail.** A read tool that ignored tenant scope would leak cross-tenant fleet +data. Guardrail: MCP reads go through the same PEP tenant scoping and DB RLS as the UI — the MCP +layer has no privileged data path. + +### F7.2 — MCP write tools mapped 1:1 to the closed vocabulary + +**What it is.** Write tools that map one-to-one to the closed verb vocabulary, carry +`destructiveHint: true` and correct `idempotentHint`, and are enforced server-side because +annotations are only hints. + +**How it works.** +1. Each write tool (`intent.gitops-open-pr` first, later `intent.argocd-sync`, + `intent.rollout-promote`, `intent.deployment-scale`) corresponds to exactly one vocabulary + verb. +2. The tool declares annotations for client UX, but the server does not trust them — it runs the + full PEP pipeline (E5) regardless. +3. There is no write tool outside the closed vocabulary; there is no generic "apply" or "exec" + tool to call. +4. `intent.gitops-open-pr` is the only write enabled first; others follow per-workspace after the + PR path is proven. + +```mermaid +flowchart TD + A["MCP write tool call (e.g. intent.gitops-open-pr)"] --> B["Annotations (destructiveHint/idempotentHint) = client UX hints only"] + B --> C["Server does NOT trust annotations"] + C --> D["Run full PEP pipeline (E5) — same as UI"] + D --> E{"1:1 with a closed-vocabulary verb?"} + E -- "no" --> R["No such tool / refuse"] + E -- "yes" --> F["Proceed under governance (approval, dispatch, audit)"] +``` + +**Acceptance criteria.** +- Write tools map 1:1 to closed-vocabulary verbs; there is no generic apply/exec tool. +- Enforcement is server-side; annotations are treated as hints only. + +**Key risk / guardrail.** A malicious or buggy client could mislabel a destructive tool as +read-only to dodge a confirmation. Guardrail: the server enforces at the PEP regardless of +annotations — the spec is explicit that annotations are not guarantees. + +### F7.3 — Elicitation-based approval on writes + +**What it is.** Write tools require Elicitation-based approval (the 2025-06-18 MCP primitive) +bound to a hash of the resolved args, and elicitation is never used to request secrets. + +**How it works.** +1. When a write needs approval, the server issues an `elicitation/create` request with a JSON + schema describing the approval, bound to the resolved-args hash (F5.9). +2. The client presents it to the user, who approves the specific action. +3. The approval is single-use; before dispatch the server re-checks the args hash. +4. Elicitation requests structured approval only — never credentials or other sensitive data, per + the spec's constraint. + +```mermaid +sequenceDiagram + autonumber + participant AG as MCP client (agent) + participant MCP as Sith MCP server + participant U as User + MCP->>MCP: compute resolved-args hash (F5.9) + MCP->>AG: elicitation/create {schema, bound to arg-hash} (never requests secrets) + AG->>U: present approval request + U-->>AG: approve (single-use, this hash) + AG-->>MCP: approval + MCP->>MCP: re-check args hash == approved hash + alt mismatch + MCP-->>AG: refuse (approve-then-swap blocked) + else match + MCP->>MCP: dispatch under governance + end +``` + +**Acceptance criteria.** +- Writes require single-use elicited approval bound to the resolved-args hash. +- Elicitation never requests credentials or sensitive data. + +**Key risk / guardrail.** Eliciting sensitive data, or a reusable approval, would create a leak or +a bypass. Guardrail: approvals are arg-hash-bound and single-use, and elicitation is limited to +structured approval — never a vehicle for secrets. + +### F7.4 — External agent as governed client + +**What it is.** Any external agent — Claude Code, Codex, kagent — is a client of the same +governance, with no privileged path: it gets exactly the governance a human does. + +**How it works.** +1. The agent connects as an MCP client and can call read and write tools. +2. Every call runs the same PEP pipeline and Ardur PDP as the UI; the agent identity ceiling is + strictly below the human's (F5.4). +3. Writes require the same elicited, arg-hash-bound approval; the agent never holds a cluster + credential. +4. Everything the agent does is audited and decision-ledgered identically to a human action. + +```mermaid +sequenceDiagram + autonumber + participant EXT as Claude Code / Codex / kagent + participant MCP as Sith MCP server + participant PEP as PEP + Ardur PDP + participant AUD as Audit + decision ledger + EXT->>MCP: read or write tool call + MCP->>PEP: SAME pipeline as UI (no privileged path) + PEP->>PEP: role/scope/verb/args/PDP/approval (ceiling below human) + PEP->>AUD: audit + decision-ledger (identical to human) + PEP-->>MCP: result / refusal + MCP-->>EXT: result / refusal +``` + +**Acceptance criteria.** +- An external MCP client issuing a read and a write is subject to identical governance (scope, + approval, audit). +- The agent holds no cluster credential and its ceiling is below the human's. + +**Key risk / guardrail.** An agent path that bypassed the PEP would be the shortest route from a +prompt to a fleet action. Guardrail: the MCP server is a thin client onto the one PEP; there is no +agent-only route, and the agent inherits every gate a human faces. + +### F7.5 — AI safety rules + +**What it is.** The behavioral rules baked into the agent surface: ground-or-abstain, evidence +before a write proposal, per-actor token/action budgets, and write proposals rate-limited +separately from reads. + +**How it works.** +1. Any statement about live state must be backed by a tool result or flagged as general knowledge + (ground-or-abstain). +2. A write may be proposed only from an evidence-citing chain; low confidence yields "here's what + I'd check", not a write. +3. Per-tenant/per-actor token and action budgets bound how much an agent can do. +4. Write proposals are rate-limited separately from (and more tightly than) reads. + +```mermaid +flowchart TD + A["Agent wants to act / assert"] --> B{"Claim backed by a tool result?"} + B -- "no" --> C["Flag as general knowledge or abstain"] + B -- "yes" --> D{"Evidence sufficient for a write?"} + D -- "no" --> E["Propose checks, not a write"] + D -- "yes" --> F{"Within token/action budget + write rate-limit?"} + F -- "no" --> G["Refuse / defer (budget/limit)"] + F -- "yes" --> H["Propose write (still gated by PEP + approval)"] +``` + +**Acceptance criteria.** +- Live-state claims are grounded in tool results or flagged; low-confidence never yields a write + proposal. +- Per-actor budgets and separate write rate-limits are enforced. + +**Key risk / guardrail.** An ungrounded or runaway agent could flood the write surface with +plausible-but-wrong proposals. Guardrail: ground-or-abstain plus separate, tighter write +rate-limits and budgets bound both the quality and the volume of what an agent can propose — and +every proposal still faces the full PEP. + +### E7 exit criteria + +- Read tools return workspace-scoped answers with `readOnlyHint`; write tools map 1:1 to the + closed vocabulary with server-side enforcement. +- Writes require single-use, arg-hash-bound elicited approval; elicitation never requests secrets. +- An external MCP client is governed identically to a human (same PEP/PDP, ceiling below human, no + credential, fully audited). +- AI safety rules (ground-or-abstain, evidence-gated writes, budgets, separate write rate-limits) + are enforced. + +## E8 — Operator console (UI) + +> **Reshape note:** the console is **one web frontend** served by both `sith ui` (local, +> single-user, kubeconfig-direct — the day-0 "fleet IDE") and `sith hub` (multi-user, governed). +> The local fleet view (E11) and this hub console render the same source-abstract fleet model. + +**Goal:** a thin, unprivileged operator console — fleet view, a workspace/cluster/service picker, +intent proposal with plan preview, multi-approver approval, and run/wave status — that is a +client of the governed API with no privileged path. + +**Phase:** P1 (a thin read view) → P3 (proposal, approval, and wave status). **Depends on:** E2 +(reads), E5 (proposal/approval flows), E6 (status and history). Per ADR-0002 the UI is +deliberately minimal; the product's value is the governed API, and the UI has exactly the +governance the MCP surface does. + +**Features:** F8.1 fleet view · F8.2 workspace/cluster/service picker · F8.3 intent proposal UX · +F8.4 multi-approver approval UX · F8.5 run/wave status view. + +### F8.1 — Fleet view + +**What it is.** A view of inventory and health across the workspace's clusters, with freshness +badges so coverage gaps are visible. + +**How it works.** +1. The view calls the read API (E2), which is tenant-scoped, and renders inventory and health per + cluster. +2. Each cluster and record shows a freshness badge; stale clusters are marked (F2.5). +3. Cross-cluster correlation results (for example "clusters where `payments` is Degraded") render + as a single fleet-wide answer. +4. The view is read-only and holds no privileged path — it only shows what the API returns for the + actor's workspace. + +```mermaid +flowchart TD + A["Operator opens fleet view"] --> B["Call read API (tenant-scoped, E2)"] + B --> C["Render inventory + health per cluster"] + C --> D["Freshness badge per cluster (stale flagged, F2.5)"] + C --> E["Cross-cluster correlation shown as one answer"] + D --> F["Coverage gaps visible, not hidden"] +``` + +**Acceptance criteria.** +- The fleet view renders tenant-scoped inventory/health with per-cluster freshness badges. +- Stale clusters are visibly flagged. + +**Key risk / guardrail.** A UI that hid staleness would present a false-complete picture. +Guardrail: freshness badges surface staleness in the view, mirroring the API's honesty about +coverage. + +### F8.2 — Workspace/cluster/service picker + +**What it is.** Tenant-scoped navigation to pick a workspace, then a cluster, then a service — +showing only what the actor is a member of. + +**How it works.** +1. The picker lists only the workspaces the actor is a member of (from signed-token claims, E1). +2. Selecting a workspace scopes everything downstream to it. +3. Within a workspace, the operator narrows to a cluster and then a service. +4. There is no way to select a workspace or cluster outside the actor's membership. + +```mermaid +flowchart TD + A["Picker opens"] --> B["List workspaces from actor's membership claims (E1)"] + B --> C["Select workspace -> scope everything to it"] + C --> D["Select cluster (within workspace)"] + D --> E["Select service (within cluster)"] + B --> F{"Workspace outside membership?"} + F -- "not shown / not selectable" --> B +``` + +**Acceptance criteria.** +- The picker shows only workspaces the actor belongs to; selection scopes all downstream views. +- No out-of-membership workspace or cluster is selectable. + +**Key risk / guardrail.** Exposing non-member workspaces in the picker would leak their existence. +Guardrail: the list derives from membership claims and is tenant-scoped server-side; the picker +cannot reach beyond it. + +### F8.3 — Intent proposal UX + +**What it is.** The flow where an operator proposes an intent — choosing a verb, a target +selector, and reviewing the dry-run plan/diff — submitted to the same governed API. + +**How it works.** +1. The operator picks a verb from the closed vocabulary and a target selector (resolved within the + workspace). +2. The UI requests a dry-run (F4.6) and shows the plan/diff before anything executes. +3. On submit, the proposal goes through the full PEP pipeline (E5) — the UI adds no privileged + path. +4. If the proposal requires approval, the UI reflects that it is pending, bound to the resolved + args. + +```mermaid +sequenceDiagram + autonumber + participant OP as Operator (UI) + participant API as Governed API + participant PEP as PEP pipeline (E5) + OP->>API: choose verb + target selector + API->>PEP: resolve target within workspace, dry-run (F4.6) + PEP-->>API: plan / diff + API-->>OP: show plan / diff before execute + OP->>API: submit proposal + API->>PEP: full pipeline (verb/args/scope/PDP/approval) + PEP-->>API: allowed / pending-approval / refused + API-->>OP: reflect status (bound to resolved args) +``` + +**Acceptance criteria.** +- An operator can propose a closed-vocabulary verb with a workspace-scoped target and preview the + plan/diff. +- Submission runs the full pipeline; the UI adds no privileged path. + +**Key risk / guardrail.** A UI that executed without a plan preview or that bypassed the pipeline +would undercut the safety model. Guardrail: dry-run precedes execute in the UI, and submission +goes through the same PEP as every other client. + +### F8.4 — Multi-approver approval UX + +**What it is.** The approval experience: an approver sees a pending action bound to its +resolved-args hash and approves it, with proposer and approver required distinct for gated +actions. + +**How it works.** +1. A gated intent appears in an approver's queue with its resolved args and the plan. +2. The approval is bound to the arg-hash (F5.9); the approver approves that specific action. +3. For multi-cluster prod, multiple distinct approvers are required (F5.5); the proposer cannot + self-approve. +4. On sufficient approvals, the action proceeds; the approval is single-use. + +```mermaid +sequenceDiagram + autonumber + participant AP as Approver (UI) + participant API as Governed API + participant PEP as PEP + API->>AP: show pending action + resolved args (arg-hash bound) + AP->>API: approve (must differ from proposer) + API->>PEP: register approval (single-use, this hash) + PEP->>PEP: enough distinct approvers? (multi-cluster prod) + alt sufficient + PEP->>PEP: proceed to dispatch + else insufficient + PEP-->>AP: still pending (await more approvers) + end +``` + +**Acceptance criteria.** +- An approver approves a specific arg-hash-bound action; the proposer cannot self-approve. +- Multi-cluster prod requires multiple distinct approvers before proceeding. + +**Key risk / guardrail.** Self-approval or a reusable approval would defeat separation of duties. +Guardrail: proposer/approver distinctness and single-use, arg-hash-bound approvals are enforced +server-side (F5.5/F5.9), not merely in the UI. + +### F8.5 — Run/wave status view + +**What it is.** A live view of a running fan-out: per-cluster outcomes, wave gates, rollback, and +any abstention message. + +**How it works.** +1. As a fan-out runs, the view shows each wave and each target's outcome (pending, succeeded, + failed). +2. Wave gates and inter-wave health checks are shown so the operator sees why the next wave has or + has not started. +3. A partial failure shows the stopped progression and the auto-rollback of the failed wave (F5.7). +4. An abstention shows the honest coverage message (for example "37/40 visible; 3 stale") rather + than a silent stop. + +```mermaid +flowchart TD + A["Fan-out running"] --> B["Show waves + per-cluster outcomes (pending/ok/failed)"] + B --> C["Show wave gates + inter-wave health checks"] + C --> D{"Partial failure?"} + D -- "yes" --> E["Show stop + auto-rollback of failed wave (F5.7)"] + D -- "no" --> F["Show progression through waves"] + A --> G{"Abstained?"} + G -- "yes" --> H["Show honest coverage message (F5.8)"] +``` + +**Acceptance criteria.** +- The view shows per-cluster outcomes, wave gates, rollback, and abstention messages in real time. +- A partial failure and its rollback are visible, and an abstention is shown honestly. + +**Key risk / guardrail.** A status view that showed only success/failure without the abstention or +rollback context would mislead the operator. Guardrail: the view surfaces gates, rollback, and the +abstention message directly from the run record (E6), so what the operator sees matches what +actually happened. + +### E8 exit criteria + +- A thin, unprivileged console renders tenant-scoped fleet view with freshness badges and a + membership-bounded picker. +- Operators propose closed-vocabulary intents with a plan preview through the same governed API. +- Approval UX enforces distinct proposer/approver and arg-hash-bound, single-use approvals + server-side. +- Run/wave status shows per-cluster outcomes, gates, rollback, and abstention truthfully. + +## E9 — Deployment and packaging + +> **Reshape note:** three things are day-one, not later. (1) **Multi-arch images** +> (`linux/amd64`+`arm64`) and **registry-relocatable** references — required for China/regulated +> estates and for arm64 laptops. (2) **cosign-signed releases + SLSA L2 provenance + SBOM** from +> the first tag. (3) The **local client** (E11) ships as a **single binary via `brew`/package +> managers** — that install path is the adoption funnel and belongs to this epic. + +**Goal:** package the hub as a Helm chart and the spoke agent as an OCM addon, support light and +heavy deployment profiles and air-gapped/on-prem installs, and define an upgrade path with an +ADR-gated addon version policy. + +**Phase:** M0 (addon enablement in the lab) → P1 (hub chart) → ongoing. **Depends on:** E0. The +control plane is a single Go binary by design (ADR-0002), which keeps packaging and supply-chain +hardening simple; secrets are referenced from a KMS, never baked into rendered output (E3). + +**Features:** F9.1 hub Helm chart · F9.2 OCM addon / spoke-agent packaging · F9.3 deployment +profiles (light vs heavy) · F9.4 air-gap / on-prem installation · F9.5 upgrade path and addon +version policy. + +### F9.1 — Hub Helm chart + +**What it is.** A Helm chart that installs the Sith hub — the control-plane binary, its +PostgreSQL dependency, configuration, and KMS references for secrets. + +**How it works.** +1. The chart deploys the control-plane binary and wires it to a PostgreSQL instance configured for + RLS (a non-owner app role, F1.5). +2. Secrets are provided as KMS references, not literal values; rendered output never contains a + secret (F3.6). +3. Configuration covers the OCM connection, KMS endpoint, and policy/PDP wiring. +4. The chart supports both profiles (F9.3) via values. + +```mermaid +flowchart TD + A["helm install sith-hub"] --> B["Deploy control-plane binary"] + A --> C["Provision / connect PostgreSQL (non-owner app role, RLS)"] + A --> D["Config: OCM connection, KMS endpoint, PDP wiring"] + A --> E["Secrets as KMS references (never literals in rendered output)"] + B --> F["Hub running, ready to federate"] + C --> F + D --> F + E --> F +``` + +**Acceptance criteria.** +- `helm install` brings up the hub with Postgres (non-owner role, RLS) and correct config. +- No secret literal appears in rendered chart output; secrets are KMS references. + +**Key risk / guardrail.** A chart that rendered secrets into manifests committed to git would leak +them (a predecessor-class failure). Guardrail: secrets are KMS references only, and repo hygiene +(F3.6) keeps rendered output free of sensitive values. + +### F9.2 — OCM addon / spoke-agent packaging + +**What it is.** The Sith spoke agent packaged as an OCM addon so the hub distributes it to spokes +through the OCM addon framework, alongside the pinned `cluster-proxy` and `managed-serviceaccount` +addons. + +**How it works.** +1. The Sith spoke agent (local allowlist + local identity, E4) is packaged as an OCM addon. +2. The hub uses the OCM addon framework to install and manage it on each registered spoke. +3. It is versioned alongside the pinned OCM addons (`cluster-proxy` v0.10.0, + `managed-serviceaccount` v0.10.0). +4. The spoke agent's local allowlist ships with it and is managed per spoke. + +```mermaid +flowchart TD + A["Sith spoke agent packaged as OCM addon"] --> B["Hub: OCM addon framework"] + B --> C["Install on spoke-a"] + B --> D["Install on spoke-b"] + C --> E["Spoke agent: local allowlist + local identity (E4)"] + D --> E + F["Pinned OCM addons: cluster-proxy v0.10.0, managed-serviceaccount v0.10.0"] --> B +``` + +**Acceptance criteria.** +- The spoke agent installs on registered spokes via the OCM addon framework. +- Addon versions are pinned and managed with the OCM addons. + +**Key risk / guardrail.** A spoke agent whose local allowlist could be silently changed from the +hub would weaken the independent second bound. Guardrail: the local allowlist is managed as part +of the spoke's own configuration (defense-in-depth), and addon versions are pinned. + +### F9.3 — Deployment profiles (light vs heavy) + +**What it is.** Two profiles from one chart: a light profile for development/lab (single binary, +minimal dependencies) and a heavy profile for production (HA, external Postgres, cloud KMS). + +**How it works.** +1. The light profile runs the single-binary hub with a minimal Postgres, suitable for `kind`/`k3d` + and demos. +2. The heavy profile runs the hub with high availability, an external managed Postgres, and a + cloud KMS. +3. The same governance and isolation apply in both; the difference is scale and dependency + externalization, not policy. +4. Values select the profile; nothing safety-relevant is disabled in the light profile. + +```mermaid +flowchart TD + A["Chart values: profile?"] --> B["Light: single binary, minimal Postgres (dev/lab)"] + A --> C["Heavy: HA hub, external Postgres, cloud KMS (prod)"] + B --> D["Same governance + isolation"] + C --> D + D --> E["Difference is scale/dependencies, not policy"] +``` + +**Acceptance criteria.** +- Both profiles deploy from one chart; the light profile suits dev/lab and the heavy profile suits + production. +- No safety control is disabled in the light profile. + +**Key risk / guardrail.** A light profile that quietly turned off RLS or KMS to "just work" in dev +would train unsafe habits and mask bugs. Guardrail: safety controls are identical across profiles; +only scale and dependency externalization change. + +### F9.4 — Air-gap / on-prem installation + +**What it is.** Installation in air-gapped or on-prem environments with no outbound internet — +mirrored images and offline addon bundles. + +**How it works.** +1. All images (hub, spoke agent, OCM addons) are mirrored to an internal registry. +2. OCM addon bundles are provided offline so enablement needs no external pulls. +3. The KMS is an on-prem/HSM equivalent reachable within the environment. +4. Install proceeds with no external network dependency. + +```mermaid +flowchart TD + A["Air-gapped environment (no outbound internet)"] --> B["Mirror all images to internal registry"] + A --> C["Provide OCM addon bundles offline"] + A --> D["On-prem KMS / HSM reachable internally"] + B --> E["helm install from internal registry"] + C --> E + D --> E + E --> F["Hub + spokes run with no external pulls"] +``` + +**Acceptance criteria.** +- The hub and spoke agents install and run with no outbound internet, from mirrored images and + offline addon bundles. +- Custody works against an on-prem KMS/HSM. + +**Key risk / guardrail.** A hidden external dependency (an image or addon pulled at runtime) would +break air-gapped installs and could be a supply-chain surprise. Guardrail: all images and addon +bundles are mirrored/offline, and the install is validated with no outbound access. + +### F9.5 — Upgrade path and addon version policy + +**What it is.** A defined upgrade path for the hub and spoke agents, with schema migrations and a +rollback, and an ADR-gated policy for bumping OCM addon versions. + +**How it works.** +1. Hub upgrades run forward schema migrations; a rollback path restores the prior version. +2. Spoke-agent addon upgrades roll out through the OCM addon framework. +3. Bumping a pinned OCM addon version is an ADR-level decision (ADR-0001 update policy), not a + silent change. +4. Upgrades preserve isolation and audit integrity (hash chain continuity, E6). + +```mermaid +flowchart TD + A["Upgrade requested"] --> B["Hub: run forward schema migration"] + B --> C{"Migration healthy?"} + C -- "no" --> D["Roll back to prior version"] + C -- "yes" --> E["Roll out spoke-agent addon upgrade"] + F["OCM addon version bump?"] --> G["ADR-gated decision (not silent)"] + E --> H["Isolation + audit hash-chain preserved"] +``` + +**Acceptance criteria.** +- Hub and spoke upgrades apply with schema migrations and a working rollback. +- OCM addon version bumps are ADR-gated; upgrades preserve isolation and audit continuity. + +**Key risk / guardrail.** A silent addon bump could change security-relevant behavior under the +plan's assumptions. Guardrail: version bumps are ADR-gated and pinned, and upgrades are validated +to preserve isolation and the audit hash chain. + +### E9 exit criteria + +- The hub installs via Helm with RLS-configured Postgres and KMS-referenced secrets; no secret + literal in rendered output. +- The spoke agent ships as an OCM addon distributed through the addon framework, versions pinned. +- Light and heavy profiles deploy from one chart with identical safety controls. +- Air-gapped/on-prem install works with mirrored images, offline bundles, and an on-prem KMS. +- Upgrades apply with migrations and rollback; addon bumps are ADR-gated; audit continuity holds. + +## E10 — Observability and SRE for Sith itself + +**Goal:** make the hub — the crown jewel — observable and operable: metrics, tracing, and +structured logs about Sith's own behavior, SLOs with error budgets, and the hardening a fleet-wide +control plane demands. + +**Phase:** the hardening posture is day-one; the surfaces mature P1 → P3. **Depends on:** E9 +(deployment) and touches every other epic. This epic observes Sith itself — it does not store +other systems' telemetry (that would be a telemetry lake, out of scope). The hub is the +highest-value target and largest blast radius in the estate, so its own operability and hardening +are first-class. + +**Features:** F10.1 metrics · F10.2 distributed tracing · F10.3 structured logging · F10.4 SLOs +and alerting · F10.5 crown-jewel hardening. + +### F10.1 — Metrics + +**What it is.** Metrics about Sith's own health and behavior: control-plane liveness, federation +freshness, intent throughput, refusal and abstention rates, PDP latency. + +**How it works.** +1. The hub exposes metrics for scraping (control-plane health, DB, queue depths). +2. Federation metrics track per-spoke read freshness and dispatch success. +3. Governance metrics track intents proposed/allowed/denied, abstention rate, and approval + latency. +4. These describe Sith itself; Sith does not retain other systems' metric series. + +```mermaid +flowchart TD + A["Sith hub"] --> B["Control-plane metrics: liveness, DB, queues"] + A --> C["Federation metrics: per-spoke freshness, dispatch success"] + A --> D["Governance metrics: intents allowed/denied, abstention rate, approval latency"] + B --> E["Exposed for scraping (about Sith itself)"] + C --> E + D --> E + E --> F["Not a telemetry lake — no other-system series retained"] +``` + +**Acceptance criteria.** +- Sith exposes control-plane, federation, and governance metrics about itself. +- No long-term storage of other systems' metric series (scope guardrail holds). + +**Key risk / guardrail.** Accreting other systems' telemetry would drift Sith into a telemetry +lake. Guardrail: metrics describe Sith's own behavior only; federated health reads stay a bounded +cache (E2), not a series store. + +### F10.2 — Distributed tracing + +**What it is.** Traces that follow an intent's lifecycle across the PEP stages and the hub → spoke +dispatch, so a slow or failed action can be localized. + +**How it works.** +1. Each intent carries a trace/correlation id from proposal through dispatch and outcome. +2. Spans cover the PEP stages (authn → … → dispatch), the PDP call, and per-spoke execution. +3. A trace shows where time went and where a refusal or failure occurred. +4. Traces reference the intent id so they correlate with audit and decision records (E6). + +```mermaid +sequenceDiagram + autonumber + participant C as Client + participant PEP as PEP (spans per stage) + participant PDP as Ardur PDP + participant SP as Spoke agent + C->>PEP: intent (trace id assigned) + PEP->>PEP: span: authn -> role -> verb -> args -> scope + PEP->>PDP: span: PDP query + PDP-->>PEP: verdict + PEP->>SP: span: dispatch + spoke execution + SP-->>PEP: outcome + Note over PEP,SP: trace id == intent id -> correlates with audit + ledger (E6) +``` + +**Acceptance criteria.** +- An intent's lifecycle is traceable across PEP stages, the PDP call, and per-spoke execution. +- Traces correlate with audit/decision records by intent id. + +**Key risk / guardrail.** Traces that captured argument values could leak secrets or sensitive +data. Guardrail: spans carry ids and timings, not secret payloads; the sanitizer (F3.6) applies to +trace attributes too. + +### F10.3 — Structured logging + +**What it is.** Structured, sanitized logs with correlation ids and no secret material. + +**How it works.** +1. Logs are structured (machine-parseable) and carry the intent/trace/correlation id. +2. The error sanitizer (F3.6) strips tokens, keys, and sensitive identifiers before emit. +3. Log levels separate routine operation from security-relevant events (refusals, abstentions, + auth failures). +4. Logs complement, but do not replace, the tamper-evident audit-log (E6). + +```mermaid +flowchart TD + A["Event in hub / spoke agent"] --> B["Structured log record + correlation id"] + B --> C["Sanitizer: strip tokens/keys/sensitive IDs (F3.6)"] + C --> D{"Security-relevant? (refusal/abstention/auth-fail)"} + D -- "yes" --> E["Elevated level for alerting"] + D -- "no" --> F["Routine level"] + E --> G["Emit safe structured log"] + F --> G +``` + +**Acceptance criteria.** +- Logs are structured, correlated by id, and free of secret material. +- Security-relevant events are distinguishable for alerting. + +**Key risk / guardrail.** An unsanitized log line can leak a token. Guardrail: sanitization is +centralized on all emit paths (F3.6), and logs are not treated as the authoritative audit trail — +the tamper-evident ledger (E6) is. + +### F10.4 — SLOs and alerting + +**What it is.** Service-level objectives for the surfaces that matter — read freshness, dispatch +success, PDP latency — with error budgets and alerting. + +**How it works.** +1. SLOs are defined for read freshness (how current the fleet model is), dispatch success rate, + and PDP decision latency. +2. Error budgets track burn; sustained burn pages. +3. Alerts fire on security-relevant conditions too (spikes in refusals/abstentions, auth failures, + signer/KMS errors). +4. SLOs are about Sith's own reliability, since a control plane that is down or slow is itself an + operational risk. + +```mermaid +flowchart TD + A["Define SLOs: read freshness, dispatch success, PDP latency"] --> B["Track error budgets"] + B --> C{"Budget burning fast?"} + C -- "yes" --> D["Page on-call"] + C -- "no" --> E["Within budget"] + F["Security signals: refusal/abstention spikes, auth fails, KMS/signer errors"] --> G["Alert"] +``` + +**Acceptance criteria.** +- SLOs exist for read freshness, dispatch success, and PDP latency, with error budgets. +- Alerts fire on both reliability burn and security-relevant conditions. + +**Key risk / guardrail.** A control plane that degrades silently is an operational hazard for the +whole fleet. Guardrail: SLOs with error budgets and security alerting make degradation visible and +actionable rather than silent. + +### F10.5 — Crown-jewel hardening + +**What it is.** The hardening the hub demands as the highest-value target: signer-key protection, +DB isolation, supply-chain integrity (SBOM, image signing), and least-privilege for the hub's own +service identity. + +**How it works.** +1. The signing key and DEKs live in KMS/HSM (E3); access is tightly scoped and audited. +2. The DB enforces RLS with a non-owner app role (E1); the hub's own service identity is + least-privilege. +3. Supply-chain integrity: images are signed and an SBOM is produced; addon versions are pinned + and verified (threat-model S8). +4. The hub is treated as the crown jewel in threat-modeling and hardening reviews, with the + dispatch path and signer as the most protected assets. + +```mermaid +flowchart TD + A["Hub = crown jewel"] --> B["Signer key + DEKs in KMS/HSM, scoped + audited (E3)"] + A --> C["DB RLS, non-owner app role, hub identity least-privilege (E1)"] + A --> D["Supply chain: signed images + SBOM, pinned + verified addons (S8)"] + A --> E["Dispatch path + signer = most-protected assets"] + B --> F["Compromise blast radius bounded (spoke re-validation, closed vocab)"] + C --> F + D --> F + E --> F +``` + +**Acceptance criteria.** +- Signer key and DEKs are KMS/HSM-protected with scoped, audited access. +- Images are signed with an SBOM; addon versions are pinned and verified; the hub identity is + least-privilege. + +**Key risk / guardrail.** The hub is the single most valuable target; its compromise is the +worst-case scenario. Guardrail: defense-in-depth — even full hub control cannot get a shell on a +spoke or bypass spoke-side re-validation, and the closed vocabulary plus per-spoke allowlists bound +the damage (threat-model S1). + +### E10 exit criteria + +- Sith exposes metrics, traces, and structured sanitized logs about its own behavior, correlated + by intent id, with no secret leakage and no other-system series retention. +- SLOs with error budgets cover read freshness, dispatch success, and PDP latency; alerts cover + reliability and security conditions. +- Crown-jewel hardening is in place: KMS/HSM key custody, RLS + least-privilege hub identity, + signed images + SBOM, pinned/verified addons. + +## E11 — Local fleet client (the adoption wedge) + +**Goal:** ship a single-binary, day-0 local tool that renders every kubeconfig context on the +engineer's machine as one searchable fleet — "k9s for your whole fleet" — with no hub, no OCM, +no account, and no telemetry, so Sith earns adoption before it asks for governance. + +**Phase:** Phase L (day 0) · **Depends on:** E2 (the source-abstract fleet model; local mode is +E2 with a kubeconfig source). This epic is the on-ramp: it needs no OCM and does not gate on +Milestone-0. It is the reshape's centre of gravity. + +**Features:** F11.1 kubeconfig auto-detect + client-side fan-out · F11.2 cache-first fleet +render (CLI + TUI) · F11.3 local web "fleet IDE" (`sith ui`) · F11.4 cross-cluster fleet search ++ correlation · F11.5 per-pod table stakes · F11.6 no-account / no-telemetry / keychain custody. + +### F11.1 — Kubeconfig auto-detect and client-side fan-out + +**What it is.** On launch, `sith` discovers every context in the user's kubeconfig(s) and opens +a read connection to each — the local mode's source adapter for the E2 fleet model. + +**How it works.** +1. Resolve kubeconfig(s) from `$KUBECONFIG` / `~/.kube/config` and enumerate contexts. +2. For each context, honor its exec-credential plugin exactly as kubectl does (aws/gcloud/az + helpers run locally); credentials never leave the machine. +3. Start a read (informer/watch) session per reachable context; mark unreachable contexts. +4. Feed each context's facts into the shared fleet model as a `source = local-kubeconfig` cluster. + +```mermaid +flowchart TD + A["sith launches"] --> B["Enumerate kubeconfig contexts"] + B --> C{"Context reachable? (exec plugin runs locally)"} + C -- "yes" --> D["Open informer/watch read session"] + C -- "no" --> E["Mark context unreachable (surface, don't fail)"] + D --> F["Feed facts into fleet model (source = local-kubeconfig)"] +``` + +**Acceptance criteria.** +- All contexts are detected; each reachable one streams reads; unreachable ones are flagged. +- No credential or kubeconfig is copied off the machine. + +**Key risk / guardrail.** A blocking auth prompt or one dead context stalling startup. Guardrail: +per-context sessions are independent and non-blocking; an unreachable context is surfaced, never +fatal. + +### F11.2 — Cache-first fleet render (CLI + TUI) + +**What it is.** A k9s-style terminal view over the aggregated fleet that renders from a local +cache in tens of milliseconds, plus scriptable CLI verbs (`sith get … --all-clusters`). + +**How it works.** +1. Watch streams hydrate a local store; the UI reads the store first, never the API per keystroke. +2. Views (resources, health, contexts) render from cache; deltas reconcile in the background. +3. A command bar (`:`/cmd-K) offers fuzzy navigation across all clusters at once. +4. CLI verbs render the same model for scripting and SSH use. + +```mermaid +flowchart LR + W["Per-context watch streams"] --> S[("Local fleet cache")] + S --> U["TUI view (renders from cache, <100ms)"] + S --> C["CLI verbs (--all-clusters)"] + A["User keystroke / query"] --> U + U -. "async" .-> W +``` + +**Acceptance criteria.** +- Views and the command bar render under ~100 ms from cache; deltas reconcile without spinners. +- CLI verbs return the same aggregated answers as the TUI. + +**Key risk / guardrail.** Per-keystroke API round-trips (the slow-UI failure). Guardrail: the +store is the single render source; the API is only a background sync target. + +### F11.3 — Local web "fleet IDE" (`sith ui`) + +**What it is.** The same fleet model served as a local web UI on `localhost` — the visual +"Lens-but-better" surface — from the same binary's embedded frontend. + +**How it works.** +1. `sith ui` starts a localhost server binding the embedded web frontend to the local fleet model. +2. The frontend is the *same* one E8 serves in hub mode; here it runs single-user, kubeconfig-direct. +3. It offers aggregated multi-cluster views, fleet search/correlation, and per-pod table stakes. +4. It binds to loopback only; no external listener, no account, no telemetry. + +```mermaid +flowchart TD + A["sith ui"] --> B["Localhost server + embedded frontend"] + B --> C["Same source-abstract fleet model (local source)"] + B --> D{"Bind scope?"} + D -- "loopback only" --> E["Single-user, no account, no telemetry"] + D -- "external" --> X["Refused — local mode is loopback only"] +``` + +**Acceptance criteria.** +- `sith ui` serves the aggregated fleet view on localhost with no account and no telemetry. +- It reuses the same frontend as the hub console (one codebase, two modes). + +**Key risk / guardrail.** Accidentally exposing local mode on a routable interface. Guardrail: +local mode binds loopback only; serving beyond the machine is a hub-mode decision with authn. + +### F11.4 — Cross-cluster fleet search and correlation (local) + +**What it is.** The wedge's signature capability in local mode: one query across every context +("every cluster where `payments` is Degraded", "which contexts run image X"). + +**How it works.** +1. The query engine (E2's F2.3) evaluates a condition across all local-source clusters at once. +2. Results aggregate into one answer listing matching contexts, with any unreachable/stale context flagged. +3. No per-context manual switching; the operator asks once. + +```mermaid +flowchart TD + A["Query across all contexts"] --> B["Evaluate over local fleet model"] + B --> C["Aggregate matches into one answer"] + C --> D{"Any context stale/unreachable?"} + D -- "yes" --> E["Flag coverage gap in result"] + D -- "no" --> F["Return complete cross-cluster answer"] +``` + +**Acceptance criteria.** +- One query returns a correct answer over ≥ 2 kubeconfig contexts; coverage gaps are flagged. + +**Key risk / guardrail.** A silently dropped unreachable context giving a false-complete answer. +Guardrail: coverage is always surfaced (reuses E2/F2.5 staleness semantics). + +### F11.5 — Per-pod table stakes (logs, exec, port-forward, YAML) + +**What it is.** The commodity single-cluster operations whose *absence* drove the Lens exodus — +present in core so the local tool is complete, but not where Sith tries to out-feature Headlamp. + +**How it works.** +1. Logs, exec, port-forward, and YAML view/edit run as ordinary K8s API calls against the + selected context, with the user's own kubeconfig identity. +2. These are local-mode conveniences; they are **not** governed typed intents and carry no + fleet-action semantics. +3. In hub mode the *same person* acts through the governed path instead — local exec is the + user's own kubectl-equivalent, not a Sith-brokered action. + +```mermaid +flowchart TD + A["Select pod in a context"] --> B{"Action"} + B -- "logs / exec / port-forward / YAML" --> C["Direct K8s API call w/ user's kubeconfig identity"] + C --> D["Local convenience (not a governed intent)"] + B -. "fleet action" .-> E["Governed typed intent path (hub, E4/E5)"] +``` + +**Acceptance criteria.** +- Logs, exec, port-forward, and YAML edit work per context in local mode. +- These paths are clearly local conveniences, distinct from the governed action model. + +**Key risk / guardrail.** Confusing local exec with a governed fleet action. Guardrail: local +per-pod ops use the user's own identity and are never dispatched as typed intents; the closed +vocabulary and no-shell rule still bind every *governed* (hub/agent) path. + +### F11.6 — No-account, no-telemetry, keychain custody + +**What it is.** The trust promises that win the Lens-refugee audience: no login wall, no +phone-home, and any local secret kept in the OS keychain (not plaintext). + +**How it works.** +1. Local mode requires no account and starts no telemetry; there is nothing to opt out of. +2. Any secret the local tool must persist goes to the OS keychain (osxkeychain / wincred / + secret-service); a missing keychain fails loudly or encrypts at rest — never silent plaintext. +3. Kubeconfig credentials are read in place and never copied or uploaded. + +```mermaid +flowchart TD + A["Local secret to persist?"] --> B{"OS keychain available?"} + B -- "yes" --> C["Store in keychain"] + B -- "no" --> D["Fail loudly or encrypt-at-rest (never silent plaintext)"] + E["Telemetry / account?"] --> F["None — nothing to opt out of"] +``` + +**Acceptance criteria.** +- No account and no network telemetry in local mode; verified with a network check. +- Secrets never land in plaintext; the keychain fallback is fail-loud, not silent. + +**Key risk / guardrail.** A silent plaintext fallback (the gh-CLI mistake). Guardrail: the +fallback is fail-loud or encrypt-at-rest by construction. + +### E11 exit criteria + +- `brew install sith && sith` → all kubeconfig contexts detected → aggregated fleet view with + cross-cluster search in **< 10 minutes**, offline, nothing leaving the machine. +- The TUI/CLI and `sith ui` render the same fleet model; per-pod table stakes work. +- No account, no telemetry; local secrets are keychain-backed with a fail-loud fallback. +- The local source feeds the *same* E2 fleet model the hub uses (one code path above the source). + +## E12 — Connector framework + +**Goal:** generalize the day-1 hand-written tool adapters into one out-of-process, typed, +versioned connector framework — so integrations scale without the in-process, unversioned sprawl +that drowned Backstage. + +**Phase:** fast-follow (P2 → P3) · **Depends on:** E2 (read adapters feed the fleet model), E4 +(typed-action adapters host verbs). Build the day-1 six by hand first; generalize once the shape +is proven — never a premature ecosystem. + +**Features:** F12.1 out-of-process gRPC connector SDK · F12.2 the three connector kinds · F12.3 +versioning + one-canonical-connector policy · F12.4 generalize the day-1 six. + +### F12.1 — Out-of-process gRPC connector SDK + +**What it is.** Connectors run as separate subprocesses speaking a typed gRPC protocol to the +hub, so a crashing connector cannot take the hub down (the Grafana model). + +**How it works.** +1. Each connector is a subprocess the hub launches and supervises over gRPC. +2. Authors code against an SDK that hides the wire protocol; the hub owns and evolves the format. +3. A panic in a connector is isolated; the hub logs it and continues. + +```mermaid +flowchart LR + H["Sith hub"] -- "gRPC" --> C1["Connector A (subprocess)"] + H -- "gRPC" --> C2["Connector B (subprocess)"] + C1 -. "panic" .-> L["Isolated: hub logs, keeps running"] + A["Author"] --> SDK["Connector SDK (protocol hidden)"] --> C1 +``` + +**Acceptance criteria.** +- Connectors run out-of-process over gRPC; a crashing connector does not crash the hub. +- Authors implement against the SDK, not the wire protocol. + +**Key risk / guardrail.** An in-process shortcut for "just one" connector reintroducing the +crash-coupling. Guardrail: all connectors are out-of-process; no in-process host access exists. + +### F12.2 — The three connector kinds (and nothing else) + +**What it is.** Every connector is exactly one of three kinds; nothing gets arbitrary host access. + +**How it works.** +1. **Read adapter** — pulls normalized facts into the fleet model (e.g. Prometheus, Loki, Helm). +2. **Brokered read-through** — deep-links to the tool's own UI/API; never re-skins it (e.g. Grafana). +3. **Typed-action adapter** — maps a closed verb to the tool's API (e.g. `argocd.sync`). +4. A connector declares its kind; the framework refuses anything outside these three. + +```mermaid +flowchart TD + A["New connector"] --> B{"Declared kind?"} + B -- "read adapter" --> R["Pull normalized facts -> fleet model"] + B -- "brokered read-through" --> D["Deep-link to tool's own UI (no re-skin)"] + B -- "typed-action adapter" --> T["Map a closed verb -> tool API"] + B -- "anything else" --> X["Refused (no arbitrary host access)"] +``` + +**Acceptance criteria.** +- Every connector is one of the three kinds; an out-of-taxonomy connector is rejected. +- Brokered read-through deep-links only; it never re-implements a tool's UI. + +**Key risk / guardrail.** Scope creep into re-skinning (the devops-portal iframe trap). +Guardrail: the taxonomy is closed; "re-skin a tool" is not an expressible connector kind. + +### F12.3 — Versioning and one-canonical-connector policy + +**What it is.** A minor-additive protocol contract and a rule of one canonical connector per +tool — the Terraform discipline that prevents the Backstage redundancy/abandonment failure. + +**How it works.** +1. Major protocol versions delineate compatibility; minor versions are strictly additive. +2. The registry admits **one** canonical connector per target tool, with declared ownership. +3. Breaking a connector's contract is a major-version, reviewed change — never a silent minor bump. + +```mermaid +flowchart TD + A["Connector change"] --> B{"Breaking?"} + B -- "no" --> C["Minor: additive, compatible"] + B -- "yes" --> D["Major: reviewed compatibility break"] + E["New connector for tool T"] --> F{"Canonical connector for T exists?"} + F -- "yes" --> G["Improve the canonical one (no duplicate)"] + F -- "no" --> H["Register as canonical, with owner"] +``` + +**Acceptance criteria.** +- Minor protocol changes are additive; breaks require a major version and review. +- The registry holds one canonical connector per tool with a named owner. + +**Key risk / guardrail.** Overlapping half-maintained connectors (the Backstage marketplace). +Guardrail: one-canonical-per-tool is enforced at registration. + +### F12.4 — Generalize the day-1 six + +**What it is.** Refactor the hand-written Argo CD, Flux, Helm, Prometheus, Loki, and GitHub +adapters onto the framework, proving it against real integrations before opening it wider. + +**How it works.** +1. Reimplement each of the six as a framework connector of its correct kind. +2. Confirm parity with the hand-written behaviour (same facts, same verbs). +3. Only after the six pass does the framework open to further tools (demand-ranked, E-later). + +```mermaid +flowchart LR + A["Hand-written six (Argo/Flux/Helm/Prom/Loki/GitHub)"] --> B["Port each onto the framework"] + B --> C{"Behaviour parity?"} + C -- "yes" --> D["Framework proven -> open to more tools (demand-ranked)"] + C -- "no" --> E["Fix framework before generalizing"] +``` + +**Acceptance criteria.** +- All six run as framework connectors with behaviour parity. +- The framework is opened to new tools only after the six pass. + +**Key risk / guardrail.** Building the framework before proving it (premature abstraction). +Guardrail: the six are the proof; generalization waits on their parity. + +### E12 exit criteria + +- Connectors run out-of-process over a versioned gRPC protocol; a crash is isolated. +- Every connector is one of the three kinds; one canonical connector per tool. +- The day-1 six run on the framework with parity; further tools are demand-ranked, not eager. + +## E13 — Cost read-overlay + +**Goal:** give the fleet a cost dimension by *reading* OpenCost per cluster and rolling it up at +the hub into per-workspace/team views (with GPU columns) — filling the documented OSS fleet-cost +gap without building a metering or optimization engine. + +**Phase:** fast-follow (P3) · **Depends on:** E2 (cost is another fleet-fact kind). This is a +read integration; it never meters, bills, or mutates clusters. + +**Features:** F13.1 OpenCost per-cluster read adapter · F13.2 hub fleet rollup · F13.3 GPU cost +columns · F13.4 freshness + non-goal guard. + +### F13.1 — OpenCost per-cluster read adapter + +**What it is.** A read adapter that pulls per-cluster allocation from an in-cluster OpenCost (or +its metrics) into the fleet model as a `cost` fact kind. + +**How it works.** +1. Where OpenCost runs on a cluster, the adapter reads its allocation output through the E2 read path. +2. Costs are normalized into `cost` fleet facts (per workload/namespace) with source + freshness. +3. Clusters without OpenCost are simply absent from the cost view (surfaced, not faked). + +```mermaid +flowchart TD + A["Cluster with OpenCost"] --> B["Read allocation via E2 read path"] + B --> C["Normalize into cost fleet facts (source + freshness)"] + D["Cluster without OpenCost"] --> E["Absent from cost view (surfaced, not faked)"] +``` + +**Acceptance criteria.** +- Per-cluster costs are ingested as `cost` facts where OpenCost exists; gaps are surfaced. + +**Key risk / guardrail.** Inventing costs for clusters that don't report them. Guardrail: no +OpenCost → no cost fact; the gap is shown, never estimated silently. + +### F13.2 — Hub fleet cost rollup (per-workspace / per-team) + +**What it is.** The capability none of the OSS tools ship free: aggregate per-cluster costs across +the fleet into per-workspace/team rollups at the hub. + +**How it works.** +1. The hub aggregates `cost` facts across all clusters in a workspace. +2. Rollups group by team/label and respect tenant scoping (E1 isolation). +3. Each rollup carries coverage (how many clusters reported) and freshness. + +```mermaid +flowchart TD + A["cost facts across workspace clusters"] --> B["Aggregate at hub (tenant-scoped)"] + B --> C["Group by team/label"] + C --> D["Rollup with coverage + freshness"] +``` + +**Acceptance criteria.** +- A per-workspace/team fleet cost rollup is produced with coverage and freshness stamped. +- Rollups respect tenant isolation. + +**Key risk / guardrail.** A partial rollup read as complete. Guardrail: coverage is always shown. + +### F13.3 — GPU cost columns (DCGM) + +**What it is.** GPU cost/utilization columns in the fleet cost view where DCGM metrics exist — +the MLOps-relevant slice of the cost overlay. + +**How it works.** +1. Where DCGM is present, GPU efficiency/idle-cost facts are ingested alongside CPU/memory cost. +2. The fleet cost view adds GPU columns; MIG/fractional attribution is best-effort where reported. +3. Absent DCGM → no GPU columns for that cluster (surfaced). + +```mermaid +flowchart TD + A["Cluster with DCGM"] --> B["Ingest GPU efficiency/idle-cost facts"] + B --> C["Add GPU columns to fleet cost view"] + D["No DCGM"] --> E["No GPU columns (surfaced)"] +``` + +**Acceptance criteria.** +- GPU cost columns appear where DCGM exists; their absence is surfaced, not faked. + +**Key risk / guardrail.** Over-claiming per-workload GPU precision. Guardrail: attribution is +best-effort and labelled; physical-GPU-level data is not presented as per-pod truth. + +### F13.4 — Freshness and non-goal guard + +**What it is.** The guard that keeps the overlay a *read* — freshness on every cost fact and a +hard line against becoming a metering/optimization engine. + +**How it works.** +1. Every cost fact and rollup carries `observed_at`; stale cost is flagged like any fleet fact. +2. The overlay never writes to clusters, never bills, never auto-rightsizes. +3. Optimization/automation requests are routed to the tools that own them (OpenCost/Kubecost/CAST AI). + +```mermaid +flowchart TD + A["Cost request"] --> B{"Read or mutate?"} + B -- "read/rollup" --> C["Serve with freshness stamp"] + B -- "meter / optimize / rightsize" --> X["Out of scope -> defer to OpenCost/Kubecost/CAST AI"] +``` + +**Acceptance criteria.** +- Cost facts and rollups are freshness-stamped; stale cost is flagged. +- No write/meter/optimize path exists in the overlay. + +**Key risk / guardrail.** Drift into a cost-optimization product. Guardrail: the overlay is +read-only by construction; mutation is not expressible here. + +### E13 exit criteria + +- Per-cluster OpenCost is read into `cost` facts; a per-workspace/team fleet rollup exists with + coverage + freshness; GPU columns appear where DCGM exists. +- The overlay never writes, meters, or optimizes — cost is a read dimension of the fleet model. + +## 8. Roadmap map + +Epics are placed below at their center of gravity — the phase where the bulk of the work lands. +Several span more than one phase: E4 (action federation) ships `gitops.open-pr` in P2 and the +live-mutation verbs in P3; E5 (policy federation) has its seam in P1, the PDP in P2, and the +fan-out reasoning in P3; E6 (audit + ledger) audits reads in P1 and is fully populated in P2; E8 +(console) is a thin read view in P1 and grows the proposal/approval/wave UX in P3; E10 (hardening) +is a day-one posture that matures throughout. A plain line (no arrowhead) between E4 and E5 marks +that they co-develop. + +**Phase L (local mode — E11 + the MCP read tools) ships day-0 and does not gate on Milestone-0.** +It reuses E2's fleet-model code with a **local kubeconfig source** and no OCM, so adoption lands +before the hub exists. E12 (connector framework) and E13 (cost overlay) are fast-follows on top of +E2/E4. Dashed links below mark **shared code**, not a gating dependency. + +```mermaid +flowchart LR + subgraph PL["Phase L — Local mode (day 0, no OCM)"] + E11["E11 Local fleet client"] + E7r["E7 MCP read tools"] + end + subgraph M0["M0 — Falsification"] + E0["E0 OCM substrate & falsification"] + end + subgraph P1["P1 — Read federation (hub)"] + E1["E1 Tenancy & identity"] + E2["E2 Read federation (source-abstract)"] + E8["E8 Operator console"] + E9["E9 Deployment & packaging"] + E10["E10 Observability & SRE"] + end + subgraph P2["P2 — First governed write"] + E3["E3 Credential & key custody"] + E4["E4 Action federation (gitops.open-pr)"] + E5["E5 Policy federation (PEP + Ardur PDP)"] + E6["E6 Audit & decision ledger"] + end + subgraph P3["P3 — Policy federation + MCP write"] + E7["E7 Governed MCP write surface"] + end + subgraph FF["Fast-follow"] + E12["E12 Connector framework"] + E13["E13 Cost read-overlay"] + end + + E11 -. "shares fleet model" .- E2 + E7r -. "same read surface" .- E11 + E11 -. "same frontend" .- E8 + E0 --> E1 + E0 --> E9 + E1 --> E2 + E1 --> E3 + E1 --> E5 + E2 --> E4 + E3 --> E4 + E5 --- E4 + E2 --> E5 + E5 --> E6 + E1 --> E6 + E4 --> E7 + E5 --> E7 + E6 --> E7 + E2 --> E8 + E5 --> E8 + E9 --> E10 + E2 --> E12 + E4 --> E12 + E2 --> E13 +``` + +The falsification gate holds above everything: E1 onward is not funded until E0 returns yes and +ADR-0001 moves to Accepted. Sequencing discipline is never violated: read before write, PR before +mutation, exec never, prod never auto. + +--- + +## 9. Open questions for the owner + +These are decisions that shape the build and are the owner's (GR's) to make. Each is grounded in +a specific epic and is left open on purpose rather than assumed. + +1. **Ardur wiring timing (E5).** ADR-0005 allows a minimal built-in policy to stand in until Ardur + is ready, then be swapped. Do we ship the built-in stand-in for the P2 first write and swap to + Ardur later, or hold the first write until Ardur's PDP / identity-broker / decision-ledger + interfaces are stable enough to wire directly? + +2. **Token issuer and identity source (E1).** What issues the signed tokens whose claims carry + workspace membership and role — an existing OIDC provider, and which one? This fixes the + authn integration and the `memberships[workspace] → role` claim shape. + +3. **KMS/HSM reference target (E3, E9).** What is the reference KMS for the heavy profile and for + air-gapped/on-prem installs (a specific cloud KMS, plus an on-prem/HSM equivalent)? This + determines the envelope-encryption and signing integration and the on-prem story. + +4. **Git host and credential model for `gitops.open-pr` (E4, E3).** Which Git hosts does the first + write target (GitHub, GitLab, Bitbucket), and what is the narrowest credential that can open a + PR on each (a scoped app token, a GitHub App installation, a deploy key)? This is the first + real secret the hub holds. + +5. **Signer key distribution to spokes (E3, E4).** How is the hub's intent-verification public key + distributed to spokes and rotated — through an OCM object, the addon bundle, or another + channel? Spoke-side verification depends on trustworthy key distribution. + +6. **Inter-wave health definition (E5).** What defines "healthy" between waves — Argo CD + application health, an Argo Rollouts analysis run, a custom probe, or a per-workspace choice? + The wave gate is only as good as this signal. + +7. **Default staleness threshold for abstention (E2, E5).** What is the default freshness + threshold that triggers abstention (the plan uses ">10m" as an illustration), and is it + configurable per workspace? This directly tunes how often Sith abstains. + +8. **Spoke-side local allowlist ownership (E4, E9).** Who authors and manages each spoke's local + allowlist, and how is it provisioned and updated? For the second, independent bound to be real + (defense-in-depth), it should not be trivially controllable from the hub alone — confirm the + intended ownership model. + +9. **MCP server exposure and client authentication (E7).** How do external agents authenticate to + the MCP server (the same signed-token model, or per-agent registered identities), and is the + MCP server exposed only within the org boundary or beyond it? This sets the reach of the + governed gateway. + +10. **P1 UI scope (E8).** ADR-0002 allows deferring the UI behind the API and MCP surface. Do we + want any UI at P1, or is a CLI plus the MCP-read surface enough until the P3 approval/wave UX + is needed? + +11. **Tamper-evidence strength (E6).** Is an internal hash-chain sufficient for the target + compliance customers, or do we need external anchoring/notarization (for example a + transparency-log-style external witness) for the audit-log and decision-ledger? + +12. **Local-mode hero surface (E11).** Ship the k9s-style **TUI** first, the local web **"fleet + IDE"** (`sith ui`) first, or both together? The TUI is the leanest day-0 wow; the web UI is + the "Lens-but-better" surface. Which is the hero the wedge leads with? + +13. **Local→hub upgrade UX (E11, E1).** When a user graduates a kubeconfig-direct cluster to an + OCM minion, what is the migration experience — re-import, run side-by-side, or promote + in-place? This is the seam between the adoption wedge and the governed hub. + +14. **MCP read tools in local mode — auth (E7, E11).** In single-user local mode, how does a + local agent authenticate to `sith serve --mcp` — loopback trust, a short-lived local token, + or an OS-keychain-held secret? This sets the day-0 agent story and the shadow-MCP defense. + +15. **Local-mode telemetry stance (E11).** "No telemetry" is the trust promise. Do we want an + explicit, off-by-default, clearly-disclosed opt-in for anonymous usage counts later, or a + permanent hard no? The Lens backlash argues for a hard no; confirm. + diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md new file mode 100644 index 0000000..9a40fe4 --- /dev/null +++ b/docs/THREAT-MODEL.md @@ -0,0 +1,189 @@ +# Sith — Threat Model + +**Status:** planning · **Date:** 2026-07-08 + +Sith's whole value is *acting across a fleet*. That same property makes it the +highest-value target and largest potential blast radius in the estate. This threat model is +therefore not an afterthought — it is a **design input**. Where a control is a design +decision, the ADR is linked and authoritative. + +Method: asset + trust-boundary enumeration, attacker scenarios, and controls, with an +explicit section on **anti-patterns to avoid** (drawn, vendor-neutrally, from a prior +control-plane prototype's real failures). + +--- + +## 1. Assets (what an attacker wants) + +| Asset | Why it matters | Where it lives | +|---|---|---| +| **The hub's dispatch path** | Whoever controls it can issue intents fleet-wide | Sith control plane (the crown jewel) | +| **Cross-tenant fleet data** | Confidential inventory/health/CVE of many tenants | Fleet model / control-plane DB | +| **Any secret the hub holds** (e.g. Git creds for `gitops.open-pr`) | Direct access to targets | Credential vault (KMS envelope) | +| **Scoped spoke tokens / brokered identities** | Cluster reach/action | OCM MSA store; Ardur identity broker | +| **The audit-log + decision-ledger** | Forensics; tampering hides an attack | Control-plane stores | +| **The signing key for intents** | Forge an intent a spoke will trust | Hub signer (see §5) | + +## 2. Trust boundaries + +```mermaid +flowchart TB + subgraph EXT["Untrusted"] + HUMAN["Operators / attackers"] + AGENT["External AI agents (MCP clients)"] + end + subgraph HUBTB["Trust boundary: Sith Hub (crown jewel)"] + PEP["PEP + API + MCP server"] + DB[("Tenant-scoped DB + RLS")] + VAULT["KMS-envelope credential vault"] + SIGN["Intent signer"] + end + subgraph ARDURTB["Trust boundary: Ardur (PDP + identity + ledger)"] + PDP["PDP / identity broker / decision-ledger"] + end + subgraph SPOKETB["Trust boundary: each spoke (independent)"] + SPOKE["Sith spoke agent: local allowlist + local identity"] + SVC["Cluster-local services"] + end + HUMAN --> PEP + AGENT --> PEP + PEP --> DB + PEP --> VAULT + PEP --> SIGN + PEP <--> PDP + SIGN -- "signed intent (outbound tunnel)" --> SPOKE + SPOKE -- "re-validate + execute w/ local identity" --> SVC +``` + +Two properties make the boundaries defensible: +1. **The spoke does not trust the hub blindly.** It independently verifies the intent + signature *and* re-checks the intent against its **own local allowlist**, executing with + its **own scoped identity**. Compromising the hub is not automatically "execute anything + on every spoke". +2. **The hub does not hold deep cluster access.** No cluster-admin kubeconfigs in the + center; reach is via scoped MSA tokens and per-action brokered identity whose ceiling is + below the human's. + +## 3. Primary attacker scenarios & controls + +### S1 — Hub compromise (the crown-jewel scenario) +*Attacker gains code-exec or credential access on the control plane.* +- **Controls:** intents are **signed**; each spoke **independently enforces a local + allowlist** and uses its **own identity** (defense-in-depth — a forged/blind dispatch is + re-validated at the spoke and bounded by local RBAC). The write vocabulary is **closed** + (no `exec`, no arbitrary `apply`), so even full hub control cannot get a shell on a + spoke. `prod` still requires **multi-approver** and **wave gates** that a single + compromised component cannot satisfy alone. Audit + decision ledgers are **append-only**. +- **Residual risk:** the signer key is the highest-value secret — protect in KMS/HSM, + rotate, and consider spoke-side allowlists tight enough that a forged intent's damage is + bounded to already-permitted verbs/targets. + +### S2 — Cross-tenant access (isolation break) +*A member of workspace A tries to see/act on workspace B.* +- **Controls:** authz derives tenant + role from **signed token claims, never request + headers**; a **DB-level RLS backstop** enforces workspace scoping *independently of + application code*; `targetSelector` is resolved **only** within the actor's workspace. + ([ADR-0003](adr/0003-tenancy-isolation.md).) +- **This directly closes the predecessor's header-trust IDOR** (see §7). + +### S3 — Malicious / buggy intent from an AI agent +*An agent (own or external via MCP) proposes a dangerous action.* +- **Controls:** the agent is a **client of the same PEP** — no privileged path. Writes + require **Elicitation approval bound to a hash of the resolved args** (agent cannot + approve-then-swap). The agent identity ceiling is **strictly below** the human's. The + agent **never holds a cluster credential**. **Ground-or-abstain**: action proposals must + cite evidence; low-confidence → no write proposal. ([ADR-0005](adr/0005-ai-mcp-ardur-pdp.md).) + +### S4 — Compromised or malicious spoke +*A spoke agent is subverted or a spoke lies in its reports.* +- **Controls:** blast radius is bounded to that spoke's own RBAC (the hub never granted it + more). False reports degrade *that* cluster's data; **freshness + source stamping** and + **abstention** prevent one bad spoke from triggering unsafe fleet-wide action. The hub + authenticates spokes via OCM identity. + +### S5 — Fan-out gone wrong (the federation-specific risk) +*One intent hits N clusters; partial failure or stale view causes damage.* +- **Controls:** **wave/canary ordering** with a **gate per wave**; **stop-on-failure + + auto-rollback**; **idempotency/dedupe** so retries can't double-apply; + **max-clusters-per-intent** ceiling; **abstention** when the targeted set is + incomplete/stale ("37/40 visible, 3 stale >10m — refuse"). ([ADR-0004](adr/0004-typed-intent-action-model.md).) + +### S6 — Secret / key compromise +*An attacker reads a hub-held secret.* +- **Controls:** **envelope encryption via KMS with per-tenant data keys** — one leak does + **not** decrypt every tenant. No single process-wide key. Entropy floor + boot check; + secrets never rendered to logs/git. ([ADR-0006](adr/0006-credential-key-custody.md).) +- **This directly closes the predecessor's single-env-key blast radius** (see §7). + +### S7 — MCP write surface abuse +*The MCP server is the shortest path from "prompt" to "fleet action".* +- **Controls:** it is the **most-hardened** surface: server-side enforcement (annotations + are *hints*, not gates); Elicitation approval on every write; same PEP, same PDP, same + audit; separate, tighter rate limits for write proposals vs reads; `gitops.open-pr` is + the only write enabled first. + +### S8 — Supply-chain / addon trust +*OCM addons or Sith images are tampered.* +- **Controls:** pin addon versions (cluster-proxy v0.10.0, managed-serviceaccount v0.10.0); + verify images (signing/SBOM) in later phases; treat the addon surface as a dependency + with a documented update policy (an ADR gates version bumps). + +## 4. Blast-radius controls (summary) + +- **Closed verb vocabulary** (hub) **+** **spoke local allowlist** (independent) = two + bounds on every write. +- **No shell, no free-form apply, no secret/RBAC writes** — ever. +- **`prod` never auto-acts**; multi-approver + per-wave gates for fan-out. +- **Per-action, non-reusable approvals** bound to arg hashes. +- **Scoped, short-lived, below-human identity** for every execution. +- **Per-tenant key custody**; **DB-level tenant backstop**. +- **Abstention** as a first-class, logged outcome. +- **Everything audited** (what-happened) **+ decision-ledgered** (why-allowed). + +## 5. On signed intents + +- The hub **signs** every dispatched intent; each spoke **verifies** before acting. This is + the integrity anchor of action federation. +- The signer key lives in KMS/HSM, is rotated, and is the single most sensitive secret in + the system. Spoke allowlists are the compensating control if it is ever compromised. +- Approvals are separately bound (arg-hash) so a valid signature is *necessary but not + sufficient* for a gated action. + +## 6. On abstention (a safety feature, not a failure) + +An operations control plane that *guesses* is more dangerous than one that says "I don't +know." Sith treats **"I won't act"** as a first-class, logged outcome whenever: +- the targeted fleet set is **incomplete or stale**, +- evidence for an AI-proposed action is **insufficient**, +- a required **approval** is missing, or +- an arg fails schema / a verb is unknown. + +Abstention is unique to a *federated* world (a single-cluster tool cannot express "3 of my +40 clusters are dark") and is safety-critical here. + +## 7. Anti-patterns to avoid (lessons carried in, vendor-neutrally) + +A prior control-plane prototype failed multi-tenant security in specific, instructive ways. +Sith is designed to make each failure **structurally impossible**: + +| Prior anti-pattern | Consequence there | Sith's structural fix | +|---|---|---| +| **Authz from spoofable request headers** (`x-user-role` / `x-organization-id`) trusted with no re-check | Any authenticated user could read/write another tenant's config (IDOR + priv-esc) | Authz from **signed token claims only**; membership re-verified server-side ([ADR-0003](adr/0003-tenancy-isolation.md)) | +| **Advertised RLS was inert** (dead code; app connected as table owner) | No DB backstop behind app-layer scoping | **Real RLS**: non-owner role, `FORCE ROW LEVEL SECURITY`, per-request scope in a transaction — present from day one | +| **Single env master key** decrypting *every* tenant's secrets | One leak = unbounded, all-tenant compromise | **KMS envelope + per-tenant data keys** ([ADR-0006](adr/0006-credential-key-custody.md)) | +| **Shared, admin-by-default cluster credential** in the center | Confused-deputy; center compromise = cluster-admin everywhere | **No admin kubeconfig in center**; scoped MSA tokens + per-action brokered identity, re-validated locally | +| **Command injection via `sh -lc` string interpolation** | Non-admin → in-cluster RCE | **No shell path exists.** Typed verbs only; args schema-validated; execution is structured, never a shell string | +| **Fail-open "denylist of one"** for destructive tools | A forgotten classification became auto-executable | **Fail-safe allowlist**; CI test asserts every write handler is classified; unknown = refuse | +| **The danger surface was untested** (routes/exec/auth) | High-blast-radius code had ~no coverage | Governance surfaces (PEP, tenant boundary, signing, abstention) are the **primary** test targets | + +## 8. Out-of-scope threats (declared) + +- Physical security of clusters; the security of the spokes' *own* workloads; the + correctness of Argo CD/Grafana themselves — Sith federates them, it does not secure their + internals. +- DoS against a single spoke's local services (bounded to that cluster). +- Full formal verification of policy — Ardur is the policy engine; Sith enforces its + decisions. + +*Every external claim about OCM/MCP/competitors referenced here is verified and cited in +[`../COMPETITIVE.md`](../COMPETITIVE.md) and the ADRs.* diff --git a/docs/adr/0001-adopt-ocm-vs-bespoke-tunnel.md b/docs/adr/0001-adopt-ocm-vs-bespoke-tunnel.md new file mode 100644 index 0000000..66c050d --- /dev/null +++ b/docs/adr/0001-adopt-ocm-vs-bespoke-tunnel.md @@ -0,0 +1,99 @@ +# ADR-0001 — Adopt OCM as the substrate (vs. a bespoke tunnel/agent) + +**Status:** **Accepted** (Milestone-0 falsification **passed** 2026-07-08) · **Date:** 2026-07-08 + +## Context + +Sith needs a central hub to reach services inside many Kubernetes clusters that live in +isolated networks (different VPCs, behind NAT), **without** the hub holding cluster-admin +credentials and **without** requiring inbound access to each spoke. The naive instinct is +to build a bespoke "outbound-only agent + reverse tunnel + reach cluster-local services". + +This mechanism is **not** the differentiator, and it is security-sensitive infrastructure. +Web-verified as of July 2026, it already exists, hardened and maintained: + +- **OCM `cluster-proxy`** (**v0.10.0**, 2026-02-02): *"establishes reverse proxy tunnels + from the managed cluster to the hub cluster … enabling clients from the hub network to + access services in the managed clusters' network even when all the clusters are isolated + in different VPCs."* Automates `apiserver-network-proxy` (Konnectivity) on hub + spokes. +- **OCM `managed-serviceaccount`** (**v0.10.0**, 2026-02-02): syncs ServiceAccounts to + spokes and **projects scoped tokens back to the hub** — i.e., scoped identity without a + central god-credential. +- The same pattern is proven independently by **Rancher `remotedialer`** (v0.6.1) and the + **Kubernetes SIG `apiserver-network-proxy`** (Konnectivity). +- **OCM** is a **CNCF Sandbox** framework (accepted 2021-11-09); core `ocm` is **v1.3.1** + (2026-05-19). + +Building this bespoke would re-implement remotedialer/Konnectivity/cluster-proxy worse than +three funded/CNCF efforts. + +## Decision + +**Adopt OCM as the connectivity + scoped-identity substrate.** Sith depends on +`cluster-proxy` (reach cluster-local services), `managed-serviceaccount` (scoped tokens), +and the `ManagedCluster`/registration APIs. Sith builds **only** the layer above: +read/action/policy federation, governance, and the MCP surface. + +**This decision is gated by a falsification test (Milestone-0):** + +> Stand up an OCM hub + 2 local spokes (`kind`/`k3d`), enable `cluster-proxy` + +> `managed-serviceaccount`, and reach a spoke's in-cluster Grafana/Argo CD from the hub via +> a scoped MSA token. **If this works in ≤ ~1 day → the "build the transport" scope is +> deleted and this ADR is Accepted. If it does not → the premise is wrong; stop and +> re-evaluate before any product code (ADR moves to Rejected).** + +The deciding experiment, its steps, exit criteria, and demo are specified in +[`../ROADMAP.md`](../ROADMAP.md) (Milestone-0). The result is appended here as evidence. + +## Consequences + +**Positive** +- Deletes ~a year of building/hardening security-sensitive transport. +- Inherits OCM's registration, addon framework, and scoped-token model. +- Keeps the hub free of cluster-admin kubeconfigs (structural blast-radius reduction). +- Aligns with a live CNCF ecosystem (KubeStellar/KubeFlex build on the same substrate). + +**Negative / risks** +- **Dependency on OCM's roadmap and addon versions.** Mitigation: pin versions + (cluster-proxy/managed-serviceaccount v0.10.0), document an update policy (version bumps + are ADR-gated), track OCM's CNCF maturity progression. +- **Operational learning curve** for OCM (hub/klusterlet/addons). Mitigation: Milestone-0 + is exactly the cheap place to hit this. +- **OCM is Sandbox, not Graduated** — some maturity risk. Mitigation: the addons we depend + on are precisely the well-exercised ones; abstract the transport behind a thin internal + interface so an alternative (remotedialer/Konnectivity direct) remains possible. + +## Alternatives considered + +1. **Bespoke outbound tunnel + agent.** Rejected: commodity, security-sensitive, and we + would do it worse than OCM/Rancher/Konnectivity. This is the "seductive trap". +2. **Rancher `remotedialer` / Konnectivity directly.** Viable fallback, but OCM bundles + the tunnel *and* scoped-identity (`managed-serviceaccount`) *and* registration in one + framework — more of the substrate for free. Kept as an escape hatch behind an interface. +3. **Agentless (upload kubeconfigs to the hub).** Rejected outright: centralizes deep + credentials — the exact anti-pattern this product exists to avoid. + +## Falsification evidence (Milestone-0) + +- **Result: ✅ PASS.** A central OCM hub reached an in-cluster service (`nginx.sith-demo`) + on a managed spoke using a **scoped `managed-serviceaccount` token** — over the + `cluster-proxy` reverse (konnectivity) tunnel, with the spoke connecting **outbound-only** + and the hub holding **no admin kubeconfig** for the spoke. The scoping is real, not + incidental: the same token was **denied** cluster-wide `secrets` and `nodes` (403), while + the in-scope service/pod reads succeeded — reach and privilege are decoupled. Connection + tracking on the spoke showed **every** hub-directed flow originating on the spoke to the + hub kube-apiserver, and **zero** hub→spoke-initiated flows. +- **Setup time: ~15 minutes** of hands-on execution (kind clusters → registration → addons → + passing reach-test → outbound-only verification), far inside the `≤ ~1 day` exit criterion. +- **Environment:** two single-node `kind` clusters (k8s v1.34.0); `clusteradm` v1.3.1 / OCM + core v1.3.1; `cluster-proxy` **0.10.0** and `managed-serviceaccount` **0.10.0** (the pinned + versions above, confirmed latest as of 2026-07-08). +- **Consequence:** the "build the transport/agent" scope is **deleted**. Sith adopts OCM as + the connectivity + scoped-identity substrate and builds only the federation/governance + layer above it. **Proceed to Phase 1.** +- **Notes / runbook + verbatim command output:** + [`../experiments/M0-ocm-falsification.md`](../experiments/M0-ocm-falsification.md). +- One upstream papercut found and worked around (does not affect the verdict): the + `cluster-proxy` 0.10.0 Helm chart templates a `ManagedProxyConfiguration` field + (`proxyAgent.additionalValues`) its own bundled CRD does not declare — see the experiment + doc's "Caveats" for the one-line CRD workaround. Worth filing upstream. diff --git a/docs/adr/0002-stack-and-language.md b/docs/adr/0002-stack-and-language.md new file mode 100644 index 0000000..cc97db9 --- /dev/null +++ b/docs/adr/0002-stack-and-language.md @@ -0,0 +1,71 @@ +# ADR-0002 — Stack & language + +**Status:** Proposed · **Date:** 2026-07-08 + +## Context + +Sith has three distinct surfaces with different constraints: +1. **Cluster-side** (the Sith spoke agent + anything that speaks to OCM/Kubernetes APIs). +2. **Control-plane / API** (the hub: read/action/policy federation, PEP, MCP server). +3. **UI** (an operator console — modest; not the product's center of gravity). + +Constraints and facts: +- The **OCM and Kubernetes ecosystem is Go** — controller-runtime, client-go, the OCM + addon-framework, `cluster-proxy`/`managed-serviceaccount`, kubebuilder. Anything that is a + Kubernetes controller/addon or links OCM libraries is overwhelmingly least-friction in Go. +- Ardur (the PDP/runtime governance) is a **Go** project. +- The MCP ecosystem has mature SDKs in multiple languages; MCP servers are commonly written + in TypeScript or Go/Python. Enforcement must be server-side regardless of SDK. +- The team is small; **fewer languages = less cognitive load**, but the *right* language per + surface matters more than uniformity where the ecosystems pull hard (cluster-side → Go). + +## Decision + +- **Cluster-side (spoke agent, OCM/K8s integration): Go.** Non-negotiable — it lives in the + Kubernetes ecosystem, links OCM/controller-runtime, and matches Ardur. This is where the + ecosystem gravity is strongest and where correctness/security matter most. +- **Control-plane / API / MCP server: Go.** Chosen for (a) one language shared with the + cluster-side and Ardur, (b) strong concurrency for fan-out dispatch, (c) a single binary + and simple deployment, (d) a viable MCP server story. This keeps the security-critical + core in **one** language and one review surface. +- **UI: a thin TypeScript/React console**, treated as a **client** of the governed API with + **no privileged path** — it has exactly the governance the MCP surface does. Deliberately + minimal; the product's value is the governed API, not the UI. +- **Datastore: PostgreSQL** for the control-plane state (fleet model cache, workspaces, + intents, decisions, audit), chosen specifically because it supports the **row-level + security (RLS)** backstop that [ADR-0003](0003-tenancy-isolation.md) requires. A cache + (e.g. Redis) may back the fleet model / rate limits later. + +**Guiding principle:** keep the **security-critical core (PEP, action federation, spoke +agent) in Go, in one place, small and heavily reviewed**; keep the UI thin and unprivileged. + +## Consequences + +**Positive** +- One primary language (Go) across the security-critical surfaces and shared with OCM + + Ardur → less context-switching, shared libraries, one review discipline. +- Postgres RLS gives a **real DB-level tenant backstop** (a direct fix for a predecessor + anti-pattern; see ADR-0003). +- Single-binary control plane simplifies deployment and supply-chain hardening. + +**Negative / risks** +- Go's MCP server ecosystem is less mature than TypeScript's in some respects. Mitigation: + keep the MCP layer a thin adapter over the same PEP; the SDK choice is reversible because + enforcement is server-side and language-independent. +- A TS/React UI adds a second toolchain. Mitigation: keep it minimal and unprivileged; + it can even be deferred behind the API + MCP surface in early phases. + +## Alternatives considered + +- **TypeScript/Node for the control plane** (to match a rich MCP/agent ecosystem). + Rejected as the *core* language: it would split the security-critical code from the + cluster-side Go and from Ardur, doubling the review surface for the most sensitive code. + TS remains the choice for the thin UI. +- **Python for the control plane** (agent/AI ecosystem). Rejected for the core for the same + split-surface reason and weaker single-binary/deploy story; may appear only in + offline/analysis tooling, never on the enforcement path. +- **Rust** for the core. Attractive for safety, but ecosystem friction with OCM/K8s + (Go-centric) and team velocity outweigh the benefit at this stage. Revisit only if a + specific component demands it. + +*This ADR governs the eventual implementation; no code is written in the planning phase.* diff --git a/docs/adr/0003-tenancy-isolation.md b/docs/adr/0003-tenancy-isolation.md new file mode 100644 index 0000000..94b1d31 --- /dev/null +++ b/docs/adr/0003-tenancy-isolation.md @@ -0,0 +1,80 @@ +# ADR-0003 — Tenancy model & multi-tenant isolation + +**Status:** Proposed · **Date:** 2026-07-08 + +## Context + +Sith is multi-tenant: many teams/tenants share one hub, each governing its own set of +clusters. Isolation *is* the product — a control plane that can see and act across many +tenants' fleets must never leak or act across the tenant boundary. + +A prior control-plane prototype failed exactly here, in instructive ways (documented +vendor-neutrally in [`../THREAT-MODEL.md`](../THREAT-MODEL.md) §7): +- authorization derived from **spoofable request headers** with no server-side re-check → + cross-tenant IDOR + privilege escalation; +- an advertised **RLS backstop that was inert** (dead code; app connected as the table + owner) → no independent DB-level enforcement; +- app-layer scoping that covered **only some models**, with a forgotten filter one bug away + from a silent cross-tenant leak. + +These must be structurally impossible in Sith. + +## Decision + +### Tenancy model +- **`Workspace` is the single scoped tenancy object.** Everything (clusters, policies, + intents, decisions, audit, memberships, fleet facts) belongs to exactly one workspace. + Tenancy is **"a workspace over many clusters"**, never "one deployment per cluster". +- A **`Membership`** grants a subject a **role** within a workspace + (`reader | operator | approver | admin`). Cluster membership in a workspace is explicit. + +### Isolation — defense in depth (all three, from day one) +1. **Authn/authz from signed token claims only — never request headers.** Tenant and role + come from the cryptographically-verified session/token (`memberships[workspace] → role`). + Request headers are **never** trusted for identity/role/tenant; any inbound + `x-*-role`/`x-*-tenant` are stripped/ignored. +2. **Application-layer scoping on every query**, via a tenant-aware data access layer that + injects the workspace scope and **hard-fails on mismatch** — covering **all** + workspace-scoped models, not a subset. A CI guard forbids un-scoped access to + workspace-scoped tables. +3. **Database-level backstop: PostgreSQL Row-Level Security (RLS), actually enforced.** + - The app connects as a **non-owner role** (owners bypass RLS). + - `ENABLE` **and** `FORCE ROW LEVEL SECURITY` on every workspace-scoped table. + - The current workspace is set **per request inside the transaction** + (`set_config('sith.workspace_id', …, true)`), and policies check it. + - This layer catches any application-layer mistake independently — it is the backstop the + predecessor advertised but never turned on. + +### Target resolution +- An intent's `targetSelector` is resolved **only against clusters in the actor's + workspace.** Cross-workspace targets are impossible by construction, not by filter. + +## Consequences + +**Positive** +- Cross-tenant access requires defeating **three independent layers** (signed token + + app scope + DB RLS). No single bug leaks tenants. +- The most dangerous predecessor bug (header-trust IDOR) is structurally gone. +- Isolation is testable as a first-class property (see below). + +**Negative / cost** +- RLS adds operational discipline (two DB roles, per-request `set_config`, transaction + hygiene) and a small performance cost. Accepted — it is the difference between a real and + a theatrical backstop. +- Every new workspace-scoped table must be wired into all three layers; the CI guard makes + omissions fail the build. + +## Verification (isolation is a primary test target) +- A cross-workspace read/write attempt is **denied at the DB layer** even if the app layer + is deliberately bypassed in the test. +- A forged/absent token is rejected; a header-injected role has **no effect**. +- Fuzz `targetSelector` with foreign cluster IDs → always resolves to empty within-workspace. + +## Alternatives considered +- **App-layer scoping only** (no DB backstop). Rejected — this is precisely the predecessor + failure; one forgotten filter = silent cross-tenant leak with nothing behind it. +- **Hard isolation via separate DB/schema per tenant.** Stronger but heavier operationally; + revisit for an enterprise tier. RLS + app-scope is the right default for many tenants. +- **`vCluster`-style control-plane isolation per tenant.** Overkill for the control-plane + data model; relevant (if at all) only to how spokes are isolated, which is the spoke's own + concern. diff --git a/docs/adr/0004-typed-intent-action-model.md b/docs/adr/0004-typed-intent-action-model.md new file mode 100644 index 0000000..c9f5d3e --- /dev/null +++ b/docs/adr/0004-typed-intent-action-model.md @@ -0,0 +1,92 @@ +# ADR-0004 — Typed-intent action model (closed vocabulary, no shell) + +**Status:** Proposed · **Date:** 2026-07-08 + +## Context + +The write path is the most dangerous part of any cross-cluster control plane: it is the +shortest route from "a request" (human or AI) to "something changed on production clusters", +multiplied by fan-out to N clusters. A prior prototype's write path had a shell +(`sh -lc` string interpolation → command injection → in-cluster RCE) and a **fail-open +"denylist of one"** for destructive tools — a forgotten classification became silently +auto-executable. + +The market's AI-ops direction makes this worse: agents want to act. An agent on an +ungoverned write path is a loaded gun. MCP's own guidance is explicit that tool annotations +(`readOnlyHint`/`destructiveHint`) are **hints, not guarantees** — enforcement must be +server-side. + +## Decision + +**The only writes Sith performs are *typed intents* from a *closed verb vocabulary*.** There +is **no shell, no free-form `apply`, no secret/RBAC mutation** — ever, at any phase. + +### The intent +``` +Intent = { + id, workspace, actor, verb ∈ CLOSED_VOCAB, + targetSelector, # resolved ONLY within the actor's workspace (ADR-0003) + args, # typed + JSON-schema-validated per verb + justification, evidenceRefs, + signature # signed by the hub; verified independently by each spoke +} +``` + +### The closed vocabulary (initial) +``` +argocd.sync | argocd.rollback +rollout.promote | rollout.abort +deployment.scale | deployment.restart +gitops.open-pr +``` +**Permanently excluded:** `exec`/shell, arbitrary `kubectl apply`, Secret create/mutate/read, +RBAC object mutation. Adding *any* verb is an ADR-level change. + +### Enforcement rules +1. **Fail-safe allowlist, not fail-open denylist.** A verb executes only if it is explicitly + in the vocabulary with a registered, schema-validated handler. Unknown verb / invalid + args → **refuse**. A **CI test asserts every handler that reaches a write path is + classified** — a forgotten classification fails the build, not production. +2. **Structured execution, never string interpolation.** Verbs map to typed API calls + (Argo CD API, Rollouts API, scale subresource, a Git PR). No argument is ever + concatenated into a shell command — there is no shell. +3. **Dry-run first** for every verb that supports it; surface the plan/diff, then require a + separate explicit step to execute. +4. **`gitops.open-pr` is the safest first write and ships first** — it is a *proposal a + human merges*, requiring zero new standing trust; perfect for an AI and consistent with + GitOps orthodoxy. Live mutations are enabled per-workspace only after the PR path is + proven. +5. **Signed dispatch + independent spoke re-validation.** The hub signs the intent; each + spoke agent **verifies the signature** and **re-validates against its own local + allowlist**, then executes with its **own scoped local identity**. A spoke **never + blindly runs what the hub sends** — two independent blast-radius bounds (hub vocabulary + + spoke allowlist/RBAC). +6. **Fan-out is governed** (see [ADR-0005](0005-ai-mcp-ardur-pdp.md) and + [`../ARCHITECTURE.md`](../ARCHITECTURE.md) §4.3): environment gates, wave ordering with a + per-wave gate, partial-failure/auto-rollback, idempotency/dedupe, and abstention on stale + views. +7. **Everything audited + decision-ledgered** (proposed → approved → dry-run → executed). + +## Consequences + +**Positive** +- The catastrophic classes (shell RCE, arbitrary apply, secret/RBAC tamper) **cannot occur** + — they are not expressible in the model. +- Every write is typed, schema-checked, signed, re-validated, and reversible-by-design where + possible (PR-first, dry-run-first). +- The same model governs human and AI actors identically. + +**Negative / cost** +- Expressiveness is deliberately limited — some legitimate operations are simply "not a verb + yet". Accepted: adding a verb is a considered, reviewed act. This friction is the feature. +- Per-verb handlers + schemas + spoke-side allowlist entries are more work than a generic + `apply`. Accepted — genericity is exactly the danger. + +## Alternatives considered +- **Generic `kubectl apply` / raw manifest writes.** Rejected: unbounded blast radius; the + predecessor's YAML-apply route was a privilege-escalation vector even with a `Secret` + refusal. Genericity defeats the whole safety model. +- **Denylist of dangerous tools.** Rejected: fail-open; one omission = auto-executable. The + predecessor's "denylist of one" is the canonical failure. +- **Let the AI emit `kubectl`/shell and sandbox it.** Rejected: sandboxing a shell is a + losing arms race; typed verbs remove the need entirely. `exec` is permanently out. diff --git a/docs/adr/0005-ai-mcp-ardur-pdp.md b/docs/adr/0005-ai-mcp-ardur-pdp.md new file mode 100644 index 0000000..8041c8c --- /dev/null +++ b/docs/adr/0005-ai-mcp-ardur-pdp.md @@ -0,0 +1,87 @@ +# ADR-0005 — AI / MCP server surface & Ardur as the PDP + +**Status:** Proposed · **Date:** 2026-07-08 + +## Context + +Sith should let AI agents operate the fleet — but the 2026 danger is real: agents on +ungoverned control planes are the largest "prompt → fleet action" risk. The market +(Komodor, SUSE Rancher Prime — verified in [`../../COMPETITIVE.md`](../../COMPETITIVE.md)) +puts agents *on top* of the fleet. Sith inverts this: **the AI is a *client* of the +governance, not the product.** + +Verified MCP facts (July 2026): +- Tool annotations (`readOnlyHint`/`destructiveHint`/`idempotentHint`/`openWorldHint`) + shipped in the **2025-03-26** spec and are **hints, not guarantees — enforce server-side**. +- **Elicitation** (a server requesting structured user input mid-flow via `elicitation/create` + + JSON schema) shipped in the **2025-06-18** spec — the native primitive for + human-in-the-loop approval. + +Ardur (ArdurAI's runtime-governance runtime) is purpose-built to be a policy decision point, +identity broker, and decision-ledger for agent actions. + +## Decision + +### Expose Sith as a governed MCP server +- **Read tools** (`fleet.inventory`, `fleet.health`, `fleet.correlate`, `fleet.cve-search`, + …) carry `readOnlyHint: true` and hit the fleet model; scoped to the caller's workspace. +- **Write tools** map **1:1 to the closed verb vocabulary** ([ADR-0004](0004-typed-intent-action-model.md)), + carry `destructiveHint: true` (+ correct `idempotentHint`), and require **Elicitation-based + approval bound to a hash of the resolved args** (the agent cannot approve-then-swap). + `intent.gitops-open-pr` ships first. +- **Annotations are hints ⇒ enforcement is server-side.** The MCP layer is a **thin adapter + over the same PEP** the UI uses. There is no privileged agent path: an external agent + (Claude Code, Codex, kagent) gets **exactly** the governance a human does. + +### Ardur is the policy decision point (and identity broker, and decision-ledger) +At the `executeIntent` boundary, the PEP asks Ardur for every intent: +> *May `{actor}` issue `{verb}` on `{resolved targets}` in `{workspace}` right now?* +→ **allow / deny / require-approval(s)** (fan-out aware: env gates, multi-approver, caps). + +- **PDP:** replaces any hardcoded "which tools need approval" with versioned, per-tenant, + fan-out-aware policy. +- **Identity broker:** Ardur mints the **short-lived, per-action, scoped execution identity** + so the AI/agent **never holds a cluster credential** and its ceiling is **strictly below** + the human's. (Complements OCM `managed-serviceaccount` on the spoke side.) +- **Decision-ledger:** Ardur records **why** each action was allowed, complementing Sith's + **audit-log** (what happened). Together = a complete agent-action record. + +### AI safety rules (baked in) +- **Ground-or-abstain:** any statement about live state must be backed by a tool result or + flagged as general knowledge; a write may be *proposed* only from an evidence-citing chain. + Low confidence → "here's what I'd check", never a write. +- **Explicit "I won't act" is a first-class, logged outcome**, not an error (ties to + federation abstention in [ADR-0004](0004-typed-intent-action-model.md) / architecture §4.3). +- **Per-tenant/per-actor token + action budgets** in the harness; write proposals rate-limited + separately from reads. +- **The MCP write surface is the most-hardened surface in the system** (see + [`../THREAT-MODEL.md`](../THREAT-MODEL.md) §3 S7). + +### Build the seam early +A **policy hook at the `executeIntent` boundary exists from Phase 1** (returns "allow" for +reads), so Ardur drops in for Phase 2 writes without re-architecture. + +## Consequences + +**Positive** +- "**A governed MCP gateway to your whole fleet**" — a platform position, not a chatbot. Any + agent inherits the org's guardrails. +- Human and agent actions are governed by one PDP, one vocabulary, one audit + ledger. +- Structural safety: the AI never holds a credential, never gets a shell, cannot bypass the PEP. + +**Negative / risks** +- Dependency on Ardur's readiness/interfaces. Mitigation: the policy-hook seam abstracts the + PDP; a minimal built-in policy can stand in until Ardur is wired, then be replaced. +- MCP is young and evolving (Elicitation is explicitly early). Mitigation: server-side + enforcement is SDK-independent; annotation/elicitation are UX on top of hard gates. +- An MCP server is an attack surface. Mitigation: it is the most-hardened surface; same PEP; + tighter write rate limits; `gitops.open-pr`-first. + +## Alternatives considered +- **AI as a first-class actor with its own privileged path.** Rejected: creates a governance + bypass; the whole thesis is that the AI is a *client* of the governance. +- **Portal-embedded assistant only (MCP client, not server).** Rejected: lower leverage; + meeting developers in their own agent (via an MCP server) + inheriting governance is the + differentiator. +- **Trust MCP annotations as enforcement.** Rejected outright: annotations are hints; the + spec says enforce server-side. Sith enforces at the PEP. diff --git a/docs/adr/0006-credential-key-custody.md b/docs/adr/0006-credential-key-custody.md new file mode 100644 index 0000000..ab7391c --- /dev/null +++ b/docs/adr/0006-credential-key-custody.md @@ -0,0 +1,72 @@ +# ADR-0006 — Credential & key custody + +**Status:** Proposed · **Date:** 2026-07-08 + +## Context + +A cross-fleet control plane is a magnet for secrets. The predecessor's custody model was its +single largest blast radius: **one env-var master key** (`TOKEN_ENCRYPTION_KEY`) decrypted +**every** tenant's kubeconfigs and tokens — one leak (a committed Helm secret, an env dump, +an SSRF reading `/proc/self/environ`, a compromised node) meant unbounded, all-tenant +compromise. It also stored **shared, admin-by-default cluster credentials** centrally +(confused-deputy), and accepted a weak key with no entropy check. The crypto *primitive* was +fine (AES-256-GCM); the **key management** was the catastrophe. + +Sith's design already removes most central secrets by construction (ADR-0001, ADR-0005): the +hub holds **no cluster-admin kubeconfigs**; reach uses OCM `managed-serviceaccount` scoped +tokens; execution uses Ardur-brokered short-lived per-action identities re-validated locally. +But some secrets remain — e.g. **Git credentials for `gitops.open-pr`**, the **intent signing +key**, and any integration tokens the hub must hold. + +## Decision + +1. **No central cluster-admin credentials.** The hub does not store per-cluster admin + kubeconfigs. Cluster reach = scoped MSA tokens; cluster action = Ardur-brokered + short-lived identity, verified and executed **locally** by the spoke with its **own** + identity. (Structurally removes the predecessor's shared-admin blast radius.) +2. **Envelope encryption via a KMS, with per-tenant data keys.** Any secret the hub must + hold is encrypted with a **per-workspace data key**, itself wrapped by a **KMS/HSM master + key** (cloud KMS or equivalent). **There is no single process-wide key.** Compromising + one tenant's data key does **not** expose other tenants; the KMS master key never leaves + the KMS. +3. **The intent signing key lives in KMS/HSM**, is rotatable, and is treated as the + highest-value secret (see [`../THREAT-MODEL.md`](../THREAT-MODEL.md) §5). Spoke-side local + allowlists are the compensating control if it is ever compromised. +4. **Key rotation is first-class:** data keys and the signing key rotate on a schedule and + on demand; a key-ring supports decrypt-old / encrypt-new during rotation. +5. **Boot-time custody checks:** refuse to start if key material is missing, below an entropy + floor, or a placeholder; verify KMS reachability. No "changeme" ever accepted. +6. **Secrets never leak to logs/git/errors:** an error sanitizer strips tokens/keys/IPs; + secrets are never rendered into Helm output committed to git or into log lines; the + `.gitignore` pre-empts common secret files; this is a **public repo** — nothing sensitive + is ever committed. +7. **Least standing secret.** Prefer short-lived, per-action, brokered credentials over + stored long-lived ones wherever possible (`gitops.open-pr` uses the narrowest Git scope + that can open a PR; no direct-push credential). + +## Consequences + +**Positive** +- The predecessor's unbounded single-key blast radius is **structurally impossible** — + per-tenant keys + KMS wrapping bound any leak to one tenant, and the master key never + leaves the KMS. +- Removing central cluster-admin creds removes the largest standing secret entirely. +- Rotation + boot checks + sanitization close the operational leak paths that actually bit + the predecessor. + +**Negative / cost** +- KMS dependency and envelope logic add operational complexity and a small per-op latency. + Accepted — this is the difference between bounded and unbounded compromise. +- Per-tenant keys add key-management surface (rotation, lifecycle). Mitigation: a + well-tested key-ring abstraction; rotation is a primary test target. + +## Alternatives considered +- **Single application-managed key (env/secret).** Rejected — the exact predecessor failure; + unbounded all-tenant blast radius. +- **Per-tenant keys without KMS wrapping** (keys in the DB/app). Better than one key, but the + key store becomes the single point of compromise. Rejected in favor of KMS-wrapped envelope. +- **Store cluster-admin kubeconfigs centrally, encrypted.** Rejected outright — even + encrypted, centralizing deep cluster credentials is the anti-pattern the whole product + avoids (ADR-0001). +- **Full per-tenant DB/schema isolation of secrets.** Possible enterprise-tier hardening; + orthogonal to envelope encryption and revisited later (see ADR-0003). diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..3d8ae80 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,21 @@ +# Architecture Decision Records + +ADRs capture *significant, hard-to-reverse* decisions with their context and consequences. +Adding a verb to the action vocabulary, taking on an OCM addon dependency, or relaxing a +safe default is an ADR-level change. + +Format: Status · Context · Decision · Consequences · Alternatives considered. Where a +decision rests on an external fact, that fact is web-verified and cited (see also +[`../../COMPETITIVE.md`](../../COMPETITIVE.md)). + +| ADR | Title | Status | +|---|---|---| +| [0001](0001-adopt-ocm-vs-bespoke-tunnel.md) | Adopt OCM as the substrate (vs. a bespoke tunnel/agent) | Proposed | +| [0002](0002-stack-and-language.md) | Stack & language (cluster-side Go; control-plane/API/UI) | Proposed | +| [0003](0003-tenancy-isolation.md) | Tenancy model & multi-tenant isolation | Proposed | +| [0004](0004-typed-intent-action-model.md) | Typed-intent action model (closed vocabulary, no shell) | Proposed | +| [0005](0005-ai-mcp-ardur-pdp.md) | AI / MCP server surface & Ardur as the PDP | Proposed | +| [0006](0006-credential-key-custody.md) | Credential & key custody | Proposed | + +All ADRs are **Proposed** — this is the planning phase; the owner reviews before any +implementation. Milestone-0's falsification result will move 0001 to Accepted/Rejected. diff --git a/docs/experiments/M0-ocm-falsification.md b/docs/experiments/M0-ocm-falsification.md new file mode 100644 index 0000000..67b076e --- /dev/null +++ b/docs/experiments/M0-ocm-falsification.md @@ -0,0 +1,333 @@ +# Milestone-0 — OCM falsification test (evidence + runbook) + +**Status:** ✅ **PASS** · **Date:** 2026-07-08 · **Verdict owner:** falsification test per +[ROADMAP Milestone-0](../ROADMAP.md) and [ADR-0001](../adr/0001-adopt-ocm-vs-bespoke-tunnel.md) + +## Verdict + +> **A central OCM hub reached an in-cluster service on a managed spoke using a scoped +> `managed-serviceaccount` token, over the `cluster-proxy` reverse tunnel, with the spoke +> connecting outbound-only and the hub holding no admin kubeconfig.** + +The core premise of ADR-0001 holds. The "build a bespoke outbound-only agent + reverse +tunnel + reach-cluster-local-services" scope is **deleted** from Sith. We adopt OCM as the +connectivity + scoped-identity substrate and build only the federation/governance layer +above it. **ADR-0001 → Accepted. Proceed to Phase 1.** + +Hands-on execution — from `kind create cluster` to the passing reach-test and the +outbound-only verification — took **~15 minutes of wall-clock** on a laptop, far inside the +`≤ ~1 day` exit criterion. + +This maps to the Milestone-0 issues: **#2** (provision), **#3** (addons + projected scoped +token), **#4** (the deciding reach-test), **#5** (outbound-only), **#6** (record verdict). + +--- + +## What was tested (and what would have falsified it) + +| Claim under test | Would be falsified if… | Result | +|---|---|---| +| `cluster-proxy` gives the hub network reach into an isolated spoke's cluster-local services | reach requires a bespoke tunnel or inbound hub→spoke access | ✅ reached `nginx.sith-demo` on the spoke | +| `managed-serviceaccount` yields a **scoped** identity, not a god-credential | the token could act as admin / the hub needed an admin kubeconfig | ✅ token can read the service; **denied** `secrets`/`nodes` | +| The spoke connects **outbound-only** | the spoke had to expose an inbound port the hub dials | ✅ all flows are spoke→hub; **zero** hub→spoke-initiated flows | + +The test was deliberately built to fail loudly: the scoped token was granted *only* +`services/proxy` + `pods` read in one namespace, so if reach had silently depended on +broader privilege, the negative-control commands (`get secrets -A`, `get nodes`) would have +**succeeded** — they did not. + +--- + +## Topology + +``` + ┌────────────────────────┐ reverse tunnel (konnectivity) ┌──────────────────────────┐ + │ HUB (kind: hub) │ ◀───────── spoke dials OUT to hub ──────────│ SPOKE (kind: spoke1) │ + │ 172.18.0.2 │ │ 172.18.0.3 │ + │ • OCM cluster-manager │ │ • klusterlet │ + │ • cluster-proxy server │ │ • cluster-proxy agent │ + │ • MSA addon-manager │ │ • MSA agent (SA+token) │ + │ • scoped MSA token ─────────────── authenticates as ──────────────────▶│ ns sith-demo: nginx svc │ + └────────────────────────┘ └──────────────────────────┘ + Hub holds NO spoke admin kubeconfig — only the scoped token secret spoke1/sith-reader. +``` + +Both clusters are single-node `kind` clusters on the shared `kind` Docker network (mutual +IP reachability). Everything scratch lived on `/Volumes/EXTENDED` (`TMPDIR`); the system +disk was never at risk (182 GiB free after the run). + +## Versions (pinned) + +| Component | Version | +|---|---| +| `clusteradm` | `v1.3.1-0-g90bdc31` | +| OCM core (`registration-operator` on hub + spoke) | `quay.io/open-cluster-management/registration-operator:v1.3.1` | +| `cluster-proxy` addon (Helm chart / app) | **`0.10.0`** / `1.1.0` | +| `managed-serviceaccount` addon (Helm chart / app) | **`0.10.0`** / `1.0.0` | +| `kind` | `v0.30.0` | +| Kubernetes (node image) | `v1.34.0` | +| `kubectl` client | `v1.33.3` | +| `helm` | `v4.1.4` | +| Docker Engine | `29.6.1` (14 CPU / 7.65 GiB VM) | + +The addon chart versions match the ADR-0001 pin exactly (`cluster-proxy` 0.10.0, +`managed-serviceaccount` 0.10.0 — both verified as the latest published charts, 2026-07-08). + +--- + +## Runbook (reproducible) + +```bash +export TMPDIR=/Volumes/EXTENDED/tmp # keep scratch off the small system disk + +# 1) Two single-node kind clusters on the shared `kind` docker network +kind create cluster --name hub # context kind-hub (172.18.0.2) +kind create cluster --name spoke1 # context kind-spoke1 (172.18.0.3) + +# 2) Bootstrap the OCM hub +clusteradm init --wait --context kind-hub +# -> prints a `clusteradm join ...` command with a bootstrap token and +# --hub-apiserver https://127.0.0.1: (host-reachable loopback) + +# 3) Register the spoke (OUTBOUND registration). +# The join runs on the host, so its own cluster-info fetch uses the host-reachable +# loopback endpoint; --force-internal-endpoint-lookup bakes the INTERNAL hub endpoint +# (hub-control-plane:6443, resolvable to 172.18.0.2 from spoke pods) into the klusterlet. +clusteradm join \ + --hub-token "" \ + --hub-apiserver https://127.0.0.1: \ + --cluster-name spoke1 \ + --force-internal-endpoint-lookup \ + --wait --context kind-spoke1 +clusteradm accept --clusters spoke1 --context kind-hub + +# 4) Enable the two addons on the hub (agents auto-deploy to the spoke). See CAVEAT below. +helm repo add ocm https://open-cluster-management.io/helm-charts && helm repo update +helm install -n open-cluster-management-addon --create-namespace \ + managed-serviceaccount ocm/managed-serviceaccount --version 0.10.0 --kube-context kind-hub +# cluster-proxy needs a one-line CRD workaround for a 0.10.0 packaging bug (see CAVEAT): +kubectl --context kind-hub get crd managedproxyconfigurations.proxy.open-cluster-management.io -o json \ + | jq '(.spec.versions[].schema.openAPIV3Schema.properties.spec.properties.proxyAgent) |= (. + {"x-kubernetes-preserve-unknown-fields": true})' \ + | kubectl --context kind-hub replace -f - +helm install -n open-cluster-management-addon \ + cluster-proxy ocm/cluster-proxy --version 0.10.0 --skip-crds --kube-context kind-hub + +# 5) A trivial in-cluster service on the spoke +kubectl --context kind-spoke1 create namespace sith-demo +kubectl --context kind-spoke1 -n sith-demo create deployment nginx --image=nginx:1.27-alpine --port=80 +kubectl --context kind-spoke1 -n sith-demo expose deployment nginx --port=80 --name=nginx + +# 6) Mint a SCOPED identity: ManagedServiceAccount on the hub -> token projected back to hub +kubectl --context kind-hub apply -f - <<'EOF' +apiVersion: authentication.open-cluster-management.io/v1beta1 +kind: ManagedServiceAccount +metadata: { name: sith-reader, namespace: spoke1 } +spec: { rotation: {} } +EOF +# Grant it ONLY services/proxy + pods read in sith-demo on the spoke (deliberately minimal) +kubectl --context kind-spoke1 apply -f - <<'EOF' +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: { name: sith-reader-svcproxy, namespace: sith-demo } +rules: +- { apiGroups: [""], resources: ["services","services/proxy"], verbs: ["get","list"] } +- { apiGroups: [""], resources: ["pods"], verbs: ["get","list"] } +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: { name: sith-reader-svcproxy, namespace: sith-demo } +subjects: +- { kind: ServiceAccount, name: sith-reader, namespace: open-cluster-management-agent-addon } +roleRef: { kind: Role, name: sith-reader-svcproxy, apiGroup: rbac.authorization.k8s.io } +EOF + +# 7) THE TEST — hub reaches the spoke's in-cluster service, through cluster-proxy, +# authenticating with the scoped MSA token (NOT an admin kubeconfig): +clusteradm proxy kubectl --cluster=spoke1 --sa=sith-reader \ + --args="get --raw /api/v1/namespaces/sith-demo/services/nginx:80/proxy/" +``` + +`clusteradm proxy kubectl` is the official OCM path that ties both addons together — its own +help states: *"Use kubectl through cluster-proxy addon. (Only supports managed service +account token as certificate.)"* It port-forwards to the hub's `cluster-proxy` server, dials +the konnectivity tunnel to the spoke's kube-apiserver, and authenticates as the `--sa` +ManagedServiceAccount. The `/services/:/proxy/` path is the spoke apiserver's +built-in service-proxy subresource, which reaches the cluster-local `nginx` Service. + +--- + +## Evidence (verbatim command output from the run) + +### E1 — Spoke registered and Available; both addons healthy + +``` +$ kubectl --context kind-hub get managedcluster +NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE +spoke1 true https://spoke1-control-plane:6443 True True 10m + +$ kubectl --context kind-hub get managedclusteraddon -A +NAMESPACE NAME AVAILABLE DEGRADED PROGRESSING +spoke1 cluster-proxy True False +spoke1 managed-serviceaccount True False + +$ clusteradm proxy health --context kind-hub +CLUSTER NAME INSTALLED AVAILABLE PROBED HEALTH LATENCY +spoke1 True True True 36.611541ms +``` + +### E2 — The projected identity is a scoped ServiceAccount token (not admin, not a kubeconfig) + +``` +# The only spoke credential present in the hub's spoke1 namespace: +$ kubectl --context kind-hub -n spoke1 get secrets +NAME TYPE DATA AGE +sith-reader Opaque 2 4m8s + +# Decoded JWT payload of that projected token: +{ + "sub": "system:serviceaccount:open-cluster-management-agent-addon:sith-reader", + "namespace": "open-cluster-management-agent-addon", + "sa": "sith-reader", + "aud": ["https://kubernetes.default.svc.cluster.local"], + "exp": 1814655286 +} + +# Scope of that identity on the spoke (kubectl auth can-i, impersonating the SA): +can-i get services/proxy -n sith-demo : yes +can-i list secrets -A : no +can-i create pods/exec -n sith-demo : no +``` + +### E3 — THE TEST: hub reaches the spoke's in-cluster nginx via cluster-proxy + scoped token + +``` +$ clusteradm proxy kubectl --cluster=spoke1 --sa=sith-reader \ + --args="get --raw /api/v1/namespaces/sith-demo/services/nginx:80/proxy/" + + + + +Welcome to nginx! +... +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+... + +``` + +The hub received the cluster-local nginx page served from inside the spoke — through the +reverse tunnel, authenticated by the scoped token. + +### E4 — Scope is real: same tunnel, in-scope read works, out-of-scope reads are refused + +``` +# In scope — list the nginx pod in sith-demo: +$ clusteradm proxy kubectl --cluster=spoke1 --sa=sith-reader --args="get pods -n sith-demo -o wide" +NAME READY STATUS ... IP NODE +nginx-68fc8fd8f-h257m 1/1 Running ... 10.244.0.11 spoke1-control-plane + +# Out of scope — cluster-wide secrets: +$ clusteradm proxy kubectl --cluster=spoke1 --sa=sith-reader --args="get secrets -A" +Error from server (Forbidden): secrets is forbidden: User +"system:serviceaccount:open-cluster-management-agent-addon:sith-reader" cannot list +resource "secrets" in API group "" at the cluster scope + +# Out of scope — nodes (an admin kubeconfig would return the node list): +$ clusteradm proxy kubectl --cluster=spoke1 --sa=sith-reader --args="get nodes" +Error from server (Forbidden): nodes is forbidden: User +"system:serviceaccount:open-cluster-management-agent-addon:sith-reader" cannot list +resource "nodes" in API group "" at the cluster scope +``` + +This is the crux: reach and privilege are **decoupled**. The hub reaches exactly what the +spoke's local RBAC grants the scoped identity, and nothing more. + +### E5 — Outbound-only: the spoke dials the hub; the hub never dials into the spoke + +`cluster-proxy` 0.10.0 uses the **PortForward** entrypoint on `kind`: the spoke's +cluster-proxy `addon-agent` establishes the reverse tunnel by dialing **out** to the hub +kube-apiserver and port-forwarding to the hub proxy-server (its proxy-agent connects to a +local `127.0.0.1:8091`). So the spoke's *only* egress is to the hub kube-apiserver. + +``` +# Spoke proxy-agent is configured to reach the proxy server via a LOCAL forward, not a dial-in: +proxy-agent args: --proxy-server-host=127.0.0.1 --proxy-server-port=8091 + +# Connection tracking on the spoke node — every hub-directed flow ORIGINATES on the spoke +# (src=10.244.0.x spoke pods) to the hub kube-apiserver (dst=172.18.0.2 dport=6443): +$ docker exec spoke1-control-plane cat /proc/net/nf_conntrack | grep dport=6443 | grep dst=172.18.0.2 +ESTABLISHED src=10.244.0.9 dst=172.18.0.2 sport=42456 dport=6443 src=172.18.0.2 dst=172.18.0.3 sport=6443 dport=42456 [ASSURED] +ESTABLISHED src=10.244.0.6 dst=172.18.0.2 sport=45608 dport=6443 src=172.18.0.2 dst=172.18.0.3 sport=6443 dport=45608 [ASSURED] +ESTABLISHED src=10.244.0.10 dst=172.18.0.2 sport=53738 dport=6443 src=172.18.0.2 dst=172.18.0.3 sport=6443 dport=53738 [ASSURED] + +# Are there ANY flows the HUB initiated INTO the spoke pod network? +$ docker exec spoke1-control-plane sh -c 'cat /proc/net/nf_conntrack | grep -E "src=172.18.0.2 dst=10.244" || echo NONE' +NONE — hub never initiates a connection into the spoke + +# Per-pod client side of the tunnel connection (spoke holds the ephemeral client port): +$ nsenter -t -n ss -tnp | grep 6443 +ESTAB 0 0 10.244.0.9:42456 172.18.0.2:6443 users:(("agent",pid=2577,fd=14)) +``` + +The spoke holds the ephemeral (client) port `42456`; the hub holds the well-known `6443`. +The spoke **dialed out**. No inbound hub→spoke port is required — the property that lets +spokes live in isolated VPCs / behind NAT. + +--- + +## Caveats and honesty notes + +- **`cluster-proxy` 0.10.0 chart packaging bug (worked around, not fatal).** The chart's + `ManagedProxyConfiguration` template unconditionally sets `spec.proxyAgent.additionalValues` + (`enableImpersonation`), but the CRD shipped in the same chart (and on `main` and the + `v0.10.0` tag) does not declare that field, so `helm install` fails with + *".spec.proxyAgent.additionalValues: field not declared in schema"*. Worked around by + patching the live CRD with `x-kubernetes-preserve-unknown-fields: true` on `proxyAgent` + and installing with `--skip-crds`. This is a **cosmetic upstream release-skew issue**, not + a transport defect — `additionalValues.enableImpersonation` drives the (unused here) + service-proxy impersonation mode; it is irrelevant to the kube-apiserver-proxy + MSA-token + path this experiment exercises. Worth filing upstream; does not affect the verdict. + (Depended-upon versions remain pinned to 0.10.0 per ADR-0001.) + +- **`clusteradm proxy kubectl` emits benign `broken pipe` / `portforward` noise** on stderr + when the wrapped kubectl finishes quickly and the port-forward is torn down. The kubectl + results themselves are correct (shown above); the noise is cosmetic. + +- **Lab RBAC was applied directly to the spoke** for speed. In the product, spoke-side RBAC + for a scoped identity is delivered as an OCM `ManifestWork` / `ClusterPermission` from the + hub (never by uploading a kubeconfig). The security property under test — *reach requires + no admin credential, and privilege is bounded by spoke-local RBAC* — is unchanged. + +- **`kind`-specific:** both clusters share one Docker network, so the "isolated VPC" is + simulated, not physically separate. What is genuinely demonstrated is the **connection + directionality** (spoke→hub only) and **scoped-identity** properties, which are transport- + and topology-independent. The `entrypoint: PortForward` mode used here needs no + LoadBalancer/Ingress on the hub, which is itself the strongest outbound-only story: the + spoke only ever needs to reach the hub's kube-apiserver. + +- **One spoke, not two.** Roadmap M0 mentions 2 spokes; the task scoped this run to hub + 1 + spoke, which is sufficient to falsify the connectivity + scoped-identity premise. Memory + headroom was ample (hub 1.59 GiB, spoke 1.19 GiB of a 7.65 GiB VM), so a second spoke is a + scale detail for Phase 1, not a gate for the M0 verdict. + +## Resource footprint + +| | Before | After | +|---|---|---| +| System disk `/` free | 193 GiB | 182 GiB | +| `/Volumes/EXTENDED` free | 1.6 TiB | 1.6 TiB | +| Docker images | 3.35 GB (pruned 1.65 GB build cache first) | 4.12 GB | +| Docker VM memory in use | — | ~2.8 GiB / 7.65 GiB | + +Clusters were deleted at teardown (`kind delete cluster --name hub --name spoke1`) to +reclaim the space. + +## Conclusion + +Every claim ADR-0001 depends on was reproduced with real command output, and the negative +controls that would have exposed a hidden admin dependency all failed closed. **OCM +`cluster-proxy` + `managed-serviceaccount` deliver outbound-only, scoped, reach-cluster- +local-services connectivity.** Sith adopts OCM and does not build a bespoke transport. + + diff --git a/docs/research/MARKET-RESEARCH-2026-07.md b/docs/research/MARKET-RESEARCH-2026-07.md new file mode 100644 index 0000000..b75477a --- /dev/null +++ b/docs/research/MARKET-RESEARCH-2026-07.md @@ -0,0 +1,283 @@ +# Sith — Market & Landscape Research (July 2026) + +**Status:** research · **Date:** 2026-07-10 · **Author identity:** ArdurAI + +This is a current (mid-2026) competitive and standards scan for Sith, written to feed epic +creation. It builds on — does not duplicate — the prior research +([`USE-CASE-AND-SHAPE.md`](USE-CASE-AND-SHAPE.md), +[`market-and-form-factor.md`](market-and-form-factor.md), +[`integrations-and-ai-governance.md`](integrations-and-ai-governance.md)) and the just-landed +specs (PRs #40–#43: the Slice-0 foundation, the F2.1 7-verb source-adapter contract, the E2 +four-lens read-federation + Investigation Brain, and the F11 local fleet UX). Where those already +established a fact with a primary source, this doc cites forward rather than re-deriving. + +**What Sith is (the thing being positioned).** A single Go binary, local-first: `sith` is a +k9s/Lens-class **multi-cluster** Kubernetes fleet client (kubeconfig-based, cache-first, offline, +no account) that grows into a local web "fleet IDE" (`sith ui`) and an optional governed hub +(`sith hub`). Its design bets, from the specs, are four: (1) **cross-cluster-by-default** fleet +views; (2) a **source-abstract 7-verb connector contract** (`discover · read · query · diff · +plan · execute · verify`); (3) a **four-lens operational graph** (LIVE / DESIRED / TIMELINE / +TELEMETRY) correlated by OpenTelemetry-semconv identity keys; and (4) a **rule-based Investigation +Brain** that turns a symptom into ranked, evidence-cited root-cause hypotheses and a *proposed* +plan — deterministic, transparent, and honest about coverage (it abstains rather than guesses). + +**Verification note.** Every external claim carries a URL. Claims drawn from a web-search summary +rather than a page I fetched line-by-line are marked *(search-surfaced)* and should be spot-checked +before they become load-bearing in a pitch; GitHub issue numbers and canonical standards URLs are +stable and treated as verified. Nothing here is invented; unverifiable items are flagged. + +--- + +## 1. Competitive landscape + +The field splits into four bands. Sith deliberately touches all four but *is* none of them: it is +a local-first multi-cluster client (band A form factor) with an investigation brain and governed +action (bands C/D value) minus the SaaS/account/LLM-autonomy baggage. + +### Band A — Local Kubernetes clients (the form factor Sith adopts) + +| Tool | What it does | Gaps vs Sith's thesis | Real-user signal | +|---|---|---|---| +| **k9s** ([repo](https://github.com/derailed/k9s)) | Terminal UI, single Go binary, reads your kubeconfig, ~90% of daily kubectl. The de-facto individually-adopted client. | **One context at a time.** No merged cross-cluster view; no desired/telemetry lenses; no RCA. | Multi-cluster is the single most-requested missing feature, repeatedly, and unbuilt: [#1006 "manage resources of multiple clusters simultaneously"](https://github.com/derailed/k9s/issues/1006), [#2730 "combine resources from multiple clusters in each view"](https://github.com/derailed/k9s/issues/2730), [#3374 "multi-cluster mode?"](https://github.com/derailed/k9s/issues/3374), [#1430 "multiple tabs"](https://github.com/derailed/k9s/issues/1430). | +| **Lens / OpenLens / FreeLens** | Electron "Kubernetes IDE." Rich single-cluster GUI. | Mandatory **account/login**, telemetry-by-default, and (in the OSS build) removal of logs/shell drove an exodus; Electron weight; still per-cluster-centric. | The trust break is the textbook cautionary tale (prior research §Lens): community forked to OpenLens then MIT-licensed **FreeLens** and did not return to the commercial product. Sith's "no account, no telemetry, permissive license" is a direct answer. | +| **Headlamp** ([kubernetes-sigs](https://headlamp.dev/)) | CNCF/SIG-UI web + desktop app, plugin system, dual-mode (desktop + in-cluster). Named the Kubernetes-Dashboard successor. | Per-cluster-centric UX; multi-cluster registration exists but the center of gravity is one cluster; no cross-cluster correlation brain; plugin-authoring tax (React). | CNCF-blessed and improving monthly — the strongest *incumbent* in the exact "dual-mode client" niche. Sith must out-*fleet* it, not out-console it. | +| **kubevious** ([repo](https://github.com/kubevious/kubevious), [cli](https://github.com/kubevious/cli)) | App-centric **config validation & assurance**: CLI + Guard (cross-manifest policy enforcement) + Dashboard. Apache-2.0. | Validation/guardrails, not fleet ops; single-app/cluster framing; no timeline/telemetry lens; no cross-cluster RCA. | Closest tool to the *config-drift* rule (R4), but as a *linter/policy* engine, not a live four-lens investigator. Complementary, not competitive. | +| **Aptakube** (closed) | The one client that already aggregates **multiple clusters simultaneously** "as if one big cluster," kubeconfig-based, nothing on the cluster. | Closed-source, paid, GUI-only, no governance, no RCA. | Proves the exact aggregated-multi-cluster demand — and that the **OSS slot is empty** (prior research §form-factor). | + +### Band B — Cluster platforms / management planes + +| Tool | What it does | Gaps vs Sith | Real-user signal | +|---|---|---|---| +| **Rancher (SUSE)** | Cluster provisioning + import (outbound `cattle-cluster-agent`), RBAC, Fleet GitOps; 2026 "Liz" agentic crew + MCP. | Heavyweight platform; provisioning-centric; agentic assist is subscription-gated, not a neutral primitive; a repricing pushed alternatives-shopping (prior research). | Validates outbound-agent + multi-cluster + AI-on-top, but as a vendor platform, not a local-first client. | +| **Portainer** ([site](https://www.portainer.io/)) | Docker+K8s management GUI, multi-environment. | **Docker-era abstractions flatten K8s** (CRDs/operators/RBAC "feel hidden"); multi-cluster is "operational, not architectural"; **no native drift detection/reconciliation**; requires a dedicated `portainer` namespace (breaks multi-tenancy); `hostPath` default loses config on reschedule; **UI performance degradation unresolved as of Feb 2026**. | Portainer's own alternatives content and 2026 write-ups catalogue these gaps ([Portainer blog](https://www.portainer.io/blog/kubernetes-dashboard); [Dokploy 2026](https://dokploy.com/blog/portainer-alternatives)). Its drift/GitOps gap is exactly Sith's DESIRED lens + R4. | +| **Devtron** ([site](https://devtron.ai/)) | OSS batteries-included K8s platform (CI/CD + GitOps + obs + security). | Breadth/platform strategy — the opposite of Sith's narrow local-first wedge; heavy install. | A "do everything" platform; validates demand for unified K8s ops but not the local-first form factor. | +| **ArgoCD UI** ([argo-cd](https://argo-cd.readthedocs.io/)) | The GitOps CD console; the DESIRED-lens exemplar and a Sith connector (not a competitor). | As a **fleet viewer** it is weak: "you completely lose access to the Argo UI as a single dashboard for your application estate" across clusters; push-model multi-cluster needs credentials + inbound network to every cluster (a honeypot); pull-model means operating *N* Argo instances; **adding clusters isn't supported in the UI (CLI only)**. | [Plural: "Where ArgoCD falls short"](https://www.plural.sh/blog/where-argocd-falls-short/). This is the cross-cluster-visibility gap Sith's read federation fills, and the credential-centralization anti-pattern Sith's kubeconfig-local / OCM-outbound design avoids. | + +### Band C — SaaS ops & AI-SRE / auto-triage (the value band — and the sharpest contrast) + +This is where the "brain" competition lives, and where Sith's *deterministic, transparent, +abstaining, local, cross-cluster* stance is most differentiated. + +| Tool | What it does | Approach | Gaps vs Sith's brain | +|---|---|---|---| +| **k8sgpt** ([repo](https://github.com/k8sgpt-ai/k8sgpt)) | CNCF Sandbox, Apache-2.0. Deterministic K8s **analyzers** turn errors into human-readable insights; an **LLM only *explains*** findings. `anonymize` flag masks names before sending to the LLM. *(search-surfaced)* | Rule-scan + LLM-explain. | Single-cluster, alert/scan-scoped; the LLM step still ships data to an external model by default ("exposing private company data to OpenAI"); no cross-cluster correlation; no governed action. The **analyzer** half is philosophically close to Sith's rule brain — Sith extends it to four lenses, cross-cluster, and no-LLM-required. | +| **HolmesGPT** ([holmesgpt.dev](https://holmesgpt.dev/)) | CNCF Sandbox, Apache-2.0, co-maintained by **Robusta + Microsoft**. Agentic **ReAct loop over 30+ observability toolsets**, read-only by design; ~40% of investigations resolve on known patterns (OOMKilled, ImagePullBackOff). *(search-surfaced)* | LLM agent iterating over tools. | The strongest K8s investigation agent — but **LLM-based and non-deterministic**; "read-only by design" is a good stance Sith shares, yet the reasoning is a black box vs Sith's cited rules. Its own docs note the honest-limits problem the whole category faces. | +| **Robusta** ([home](https://home.robusta.dev/)) | CNCF Sandbox, ~2,500★. Intercepts Prometheus alerts, attaches logs/status/metrics, AI RCA, routes to Slack/Teams/PagerDuty. *(search-surfaced)* | Alert-enrichment + AI RCA (SaaS-leaning). | Alert-triggered and notification-centric; not a fleet client; not local-first. | +| **Botkube** ([botkube.io](https://botkube.io/)) | ChatOps: kubectl from Slack/Teams, AI troubleshooting, event alerts, automated workflows. *(search-surfaced)* | Chat-driven ops + AI. | Chat surface, not a fleet view; **kubectl-from-chat is exactly the exec-from-a-bot risk** Sith's typed-intent model rejects. | +| **Cleric** ([site](https://cleric.io/)) | **Autonomous** AI SRE agent: auto service-mapping, parallel hypothesis testing with confidence tracking, continuous learning; root-causes without runbooks. *(search-surfaced)* | Autonomy-first LLM agent (commercial). | The autonomy-first end of the market — the opposite of "proposes, never executes." A live argument for Sith's boundaries-not-instructions stance. | +| **Komodor** | SaaS K8s ops + multi-agent AI-SRE ("Klaudia"), agent per cluster, MCP-extensible, sandboxed remediation. | Closed SaaS (China/air-gap excluded), diagnosis-first, per-cluster agent; no neutral OSS governed-action primitive (prior research §landscape). | The closest *product* to "operate the fleet with AI," but proprietary and not local-first. | +| **Cast AI** | GPU/cost automation that *mutates* clusters (rightsizing/spot); quote-priced; EKS/GKE/AKS/OpenShift only, no China clouds / air-gap (prior research). | Cost-optimization automation. | Different category (mutation/cost); relevant only to Sith's read-only cost overlay (E13) as a boundary. | + +**The category-defining tension in Band C:** the "**hallucination gap**" — an LLM doing root-cause +without live telemetry, topology, and recent state transitions "bridges the gap using probability, +which is why answers often sound confident even when they are wrong" +([Sherlocks.ai](https://www.sherlocks.ai/blog/the-hallucination-gap-why-general-llms-fail); +*search-surfaced*). Independent academic work is now building *falsification* methodology for +agentic Kubernetes operations precisely because reported "accuracy" is observational +([arXiv 2605.23058](https://arxiv.org/pdf/2605.23058); *search-surfaced*). Sith's brain is the +structural answer: it reasons over the **live four-lens graph**, cites the exact signals per +hypothesis, and **abstains when a required lens is missing** — it cannot hallucinate a cause it +has no evidence for, and it never sends cluster data to a third-party model. + +### Band D — IDPs / catalogs (adjacent, not competitors) + +Backstage / Port / Cortex / OpsLevel are developer portals — the category that lost the "single +pane of glass" bet and that the predecessor `devops-portal` died on (prior research §review-10). +Self-hosted Backstage runs to multiple FTEs and 6–12 months. Sith is **not** a portal; a portal +can *consume* Sith's API/MCP. Included only to hold the category line. + +### Comparison matrix + +Legend: ✅ yes · ◑ partial/limited · ❌ no · — n/a. "Local-first" = runs from your kubeconfig with +no server/account. "X-cluster" = merged cross-cluster views by default. "4-lens" = correlates +live+desired+timeline+telemetry. "RCA" = root-cause reasoning. "Transparent" = non-LLM / +explainable-by-construction. "Gov. action" = governed typed-intent writes. "Offline/no-acct" = no +telemetry, no login required. + +| Tool | Local-first | X-cluster | 4-lens | RCA | Transparent | Gov. action | OSS | Offline/no-acct | +|---|---|---|---|---|---|---|---|---| +| **k9s** | ✅ | ❌ (asked ×4) | ❌ (live only) | ❌ | ✅ | ❌ | ✅ | ✅ | +| **Lens/FreeLens** | ✅ (FreeLens) | ◑ (per-cluster) | ❌ | ❌ | ✅ | ❌ | ◑ | ◑ (Lens acct) | +| **Headlamp** | ✅ | ◑ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | +| **kubevious** | ◑ | ❌ | ◑ (desired/validate) | ◑ (config) | ✅ | ❌ | ✅ | ◑ | +| **Aptakube** | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | +| **Rancher** | ❌ | ✅ | ◑ | ◑ (Liz, LLM) | ❌ | ◑ (platform) | ◑ | ❌ | +| **Portainer** | ❌ | ◑ | ❌ (no drift) | ❌ | ✅ | ❌ | ◑ | ❌ | +| **ArgoCD UI** | ❌ | ❌ ("lose the dashboard") | ◑ (desired) | ❌ | ✅ | ◑ (sync) | ✅ | ❌ | +| **k8sgpt** | ◑ (CLI) | ❌ | ◑ (live+scan) | ◑ (analyzer+LLM) | ◑ (rules + LLM) | ❌ | ✅ | ◑ (LLM egress) | +| **HolmesGPT** | ◑ | ◑ | ◑ (telemetry-heavy) | ✅ (LLM) | ❌ (LLM) | ❌ (read-only) | ✅ | ❌ (LLM) | +| **Robusta** | ❌ | ◑ | ◑ | ✅ (LLM) | ❌ | ◑ | ◑ | ❌ | +| **Cleric** | ❌ | ◑ | ◑ | ✅ (autonomous) | ❌ | ◑ (autonomy) | ❌ | ❌ | +| **Komodor** | ❌ | ✅ | ◑ | ✅ (LLM) | ❌ | ◑ (SaaS) | ❌ | ❌ | +| **Sith (proposed)** | ✅ | ✅ | ✅ | ✅ (rule brain) | ✅ (cited rules) | ✅ (typed-intent, hub) | ✅ | ✅ | + +The bottom row is the thesis: **no incumbent occupies the full row.** k9s/Headlamp/Aptakube own the +local client but stop at LIVE-on-one-cluster; the AI-SRE band owns RCA but is LLM-based, SaaS-y, and +alert-scoped; Komodor does the whole thing but proprietary. Sith's defensibility is the +*combination*, in one local-first OSS binary. + +--- + +## 2. Category framing — is "local-first, cross-cluster-by-default, no-account" a real wedge? + +**Yes, with an honest caveat.** The evidence: + +- **The demand is articulated inside the incumbent's own tracker, repeatedly and unbuilt.** k9s has + *four* distinct open multi-cluster requests (#1006, #2730, #3374, #1430). That is a durable, + multi-year signal that the single-context ceiling is felt — and that the maintainer won't cross + it. *Honest caveat (from prior research):* the per-issue reaction counts are modest (single/low + double digits), so this is a **convenience/efficiency gap, not a screaming emergency** — the read + wedge is a beachhead, not by itself a moat. +- **The aggregated-multi-cluster experience already sells — but only closed.** Aptakube ships exactly + "many clusters as one," kubeconfig-based, and charges for it; the **OSS slot is empty**. ArgoCD + users hit the same wall from the GitOps side ("you lose the single dashboard"). +- **Local-first + no-account + permissive-license is a proven trust wedge.** The Lens→FreeLens + migration shows the audience punishes account walls and telemetry and rewards permissive OSS; k9s + shows install-and-run beats install-a-platform. The CNCF adoption barriers are complexity and + abandonment-fear, not missing features (prior research §CNCF-2024). +- **The privacy/offline angle is newly sharp in 2026 because of the AI-SRE band.** k8sgpt's default + data egress to OpenAI and the "hallucination gap" mean a **local, no-egress, deterministic** + investigator is not just a nicety — it is the enterprise-safe alternative to the whole LLM-SRE + category. "Cross-cluster-by-default" + "your data never leaves your laptop" + "it tells you *why* + and shows its work" is a positioning none of Band C can claim. + +**Verdict:** the wedge is real but must be sequenced correctly — lead with the *read* client +("k9s for your whole fleet"), and let the *brain* ("…that also tells you why payments is down, +without sending your logs to anyone") be the reason it becomes a default rather than a curiosity. +The brain is the differentiator; the local multi-cluster client is the on-ramp. + +--- + +## 3. Standards & guidelines to align with + +Aligning to these is both correctness and credibility — it is how a solo/small OSS project earns +trust and avoids reinventing security-sensitive wheels. + +| Standard / guideline | Why it matters to Sith | Authoritative source | 2026 status note | +|---|---|---|---| +| **OpenTelemetry Kubernetes semantic conventions** | Sith's `EntityRef` correlation keys (`k8s.cluster.name`, `k8s.namespace.name`, `container.image.digest`, `service.name`, …) are the join across the four lenses. Standardizing on semconv means connectors interoperate and the graph joins without heuristics. | [semconv/resource/k8s](https://opentelemetry.io/docs/specs/semconv/resource/k8s/); [k8s attribute registry](https://opentelemetry.io/docs/specs/semconv/registry/attributes/k8s/) | ⚠️ **Flag:** the k8s attributes were **promoted to release-candidate in 2026, not yet stable** ([OTel blog](https://opentelemetry.io/blog/2026/k8s-semconv-rc/)); migration is behind Collector feature gates. Sith should pin to the RC schema and track the stable promotion. | +| **Kubernetes API conventions** | The normalized fleet model and verb semantics should mirror K8s object/status/condition conventions so Sith feels native and stays forward-compatible. | [sig-architecture/api-conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md) | Stable, canonical. | +| **client-go exec-credential-plugin auth** | Local mode must honor exec plugins (`aws eks get-token`, `kubelogin`, `gke-gcloud-auth-plugin`) exactly as kubectl does; a cloud kubeconfig is *not* a self-contained credential. | [K8s auth reference](https://kubernetes.io/docs/reference/access-authn-authz/authentication/) | Stable (verified in prior research §identity). | +| **Open Cluster Management (OCM)** | The hub's outbound-only transport (`cluster-proxy` + `managed-serviceaccount`) — adopt, don't build (ADR-0001, M0 PASSED). | [open-cluster-management.io](https://open-cluster-management.io/); [cluster-proxy](https://github.com/open-cluster-management-io/cluster-proxy) | Stable; CNCF sandbox. | +| **Model Context Protocol (MCP)** | The governed agent surface (`sith serve --mcp`); annotations are hints, enforcement is server-side. Now vendor-neutral. | [modelcontextprotocol.io](https://modelcontextprotocol.io/); [LF donation, 2025-12-09](https://blog.modelcontextprotocol.io/posts/2025-12-09-mcp-joins-agentic-ai-foundation/) | MCP donated to the Linux Foundation's Agentic AI Foundation — vendor-neutral (verified in prior research §integrations). | +| **SLSA + Sigstore/cosign + SBOM** | Release integrity: cosign-sign, SLSA L2 provenance, SPDX/CycloneDX SBOM from the first tag. Scorecards weigh these before humans read code. | [slsa.dev](https://slsa.dev/); [docs.sigstore.dev](https://docs.sigstore.dev/) | SLSA L2 is now "an afternoon" via GitHub attestation + `slsa-github-generator`; Sigstore keyless (Fulcio OIDC + Rekor) removes long-lived keys ([2026 supply-chain guides](https://aquilax.ai/blog/supply-chain-artifact-signing-slsa); *search-surfaced*). | +| **CNCF project norms** | Permissive license (Apache-2.0 ✅), OpenSSF Scorecard, third-party audit expectations for security-adjacent tooling. | [cncf.io](https://www.cncf.io/); [OpenSSF Scorecard](https://github.com/ossf/scorecard) | Signing + CI hygiene outrank SBOM in Scorecard weighting (prior research §identity §6). | + +--- + +## 4. Differentiation & positioning + +### What makes Sith a *default* tool + +1. **Cross-cluster-by-default in a local client.** The one thing k9s/Headlamp won't do and only a + closed tool (Aptakube) ships. This is the install-me on-ramp. +2. **A brain that reasons over live evidence and shows its work — no LLM required, no data egress.** + The structural answer to the hallucination gap and to k8sgpt's OpenAI-by-default egress. "It + tells you *why* `payments` is down across the fleet, cites the exact signals, and abstains when + it can't confirm" is a claim no Band-C tool can make honestly. +3. **One engine, two modes; one brain, two modes.** The local advisory brain and the hub + governed-plan brain are the same rules — adoption and governance share code, so nothing is + thrown away when a team upgrades from laptop to hub. +4. **Boundaries, not instructions.** Typed-intent closed vocabulary, `plan`-never-`execute` at the + brain, no shell — the direct lesson from the Replit deletion and from Botkube-style + kubectl-from-chat. + +### The strongest wedge (and the sequence) + +**Lead:** "**k9s for your whole fleet**" — the local, cross-cluster, no-account read client +(Phase L / E11). **Land the differentiator right behind it:** the **local advisory Investigation +Brain** over the reachable lenses ("…that also tells you why, offline"). **Expand:** the hub turns +the same brain's plan into a governed typed intent. Positioning line: *the local-first fleet client +that tells you why — and, when you're ready, safely acts.* + +### Risks & objections (and the answer) + +- **"k9s/Headlamp already do this / will add multi-cluster."** They've had years; the maintainer + closed the k9s requests as not-planned; the differentiator is the *brain + governance*, not the + merged table alone. Answer: ship the brain, which they are not positioned to build. +- **"Rule-based can't compete with LLM breadth."** True for open-ended Q&A; false for the *six + failures that cause most incidents*. Answer: deterministic + cited + abstaining beats + confident-and-sometimes-wrong for operational trust; the arxiv falsification work backs this. +- **"Another dashboard."** Answer: cross-cluster + four-lens + RCA + governed action in one OSS + binary is not another dashboard; it is the row no one occupies (matrix §1). +- **"Solo-project trust discount."** Answer: permissive license (already Apache-2.0), no account, + no telemetry, cosign/SLSA/SBOM day one, and a CNCF-aligned standards posture (§3). +- **Telemetry-lake temptation.** The biggest *self-inflicted* risk. Answer: TELEMETRY is + **query-through** (derived answers only, no series retained) — the single decision that adds the + telemetry lens without becoming the store SCOPE forbids. + +### What NOT to build (anti-drift, confirmed by the landscape) + +- **Not another single-cluster console** — Headlamp/k9s own that; build the *fleet* view. +- **Not an LLM-autonomy SRE** — Cleric/Komodor own autonomy-first; Sith is deterministic + proposes. +- **Not a telemetry lake / metrics store** — read Prom/Loki/ES query-through; never retain series. +- **Not a GitOps reconciler / a portal / a scheduler / a cost-optimization mutator** — Argo, Flux, + Backstage, Karmada, Cast AI own those; Sith reads them and (for cost) overlays read-only. +- **Not kubectl-from-chat / free-form exec** — the Botkube/Replit anti-pattern; typed intents only. +- **Fluentd/Fluent-bit are not data sources** — read the sinks (ES/OpenSearch/Splunk/Loki), treat + shippers as health-only workloads. +- **Grafana is deep-link only** — brokered read-through, never re-skinned (no iframe-Grafana trap). + +--- + +## 5. Recommended epic themes → candidate epics/features + +Mapping findings to the existing backlog (epics #18–#39 from the implementation roadmap #39). Most +findings **reinforce** existing epics; two argue for **new** first-class epics. + +| Theme (from findings) | Evidence | Maps to | Recommended action | +|---|---|---|---| +| **Cross-cluster-by-default fleet client** is the on-ramp | k9s #1006/#2730/#3374/#1430; Aptakube-only; ArgoCD "lose the dashboard" | **E11 #29** (local fleet client) + **E2 #20** (F2.6 four-lens graph, F2.7 EntityRef) | Keep E11 the lead; ensure F2.6/F2.7 land the merged cross-cluster view early. | +| **Investigation Brain** is *the* differentiator vs Band C | hallucination gap; k8sgpt LLM-egress; HolmesGPT/Cleric black-box; deterministic+abstaining is the trust answer | **NEW epic E14 — Investigation Brain** (proposed in the E2 spec §3.7) | **Open E14** (label `epic`), depends on E2 (#20), E12 (#30), E4/E5 (#22/#23). Ship a **local advisory subset in Phase L** (rules R1–R6 over reachable lenses); governed-plan mode with the write path. This is the highest-leverage new epic. | +| **Four-lens graph + OTel-semconv keys** is the substrate the brain needs | OTel k8s semconv (RC); ArgoCD/Portainer drift gaps | **E2 #20** → add **F2.6** (four-lens operational graph) + **F2.7** (correlation keys / `EntityRef`) | Add these two features to E2; pin to the OTel RC schema and track stable promotion (flag §3). | +| **7-verb connector contract + integration waves** | Portainer no-drift; connector breadth (Argo/Prom/ES/logs) is what powers the rules | **E12 #30** (connector framework) | Encode the wave matrix (W1 = K8s+GitHub+Argo+Prom+ES+AWS) as E12 features; W1 is exactly the coverage R1–R6 need. Keep three connector kinds (RA/BR/TA); Grafana = brokered deep-link only. | +| **Local, no-egress, deterministic** as the privacy/trust wedge | k8sgpt OpenAI egress; Lens telemetry backlash | **E11 #29 / F11.6 #36** (no-account/no-telemetry/keychain) | Reinforce F11.6 as a *positioning* pillar, not just a feature; make "no data leaves the machine" a tested guarantee. | +| **Governed action, boundaries-not-instructions** | Replit deletion; Botkube kubectl-from-chat; Cleric autonomy | **E4 #22 / E5 #23 / E7 #25** | Unchanged — the landscape *validates* the closed-vocabulary + PEP + MCP-server design. Brain's `plan` hands off to E4/E5; MCP write tools stay elicitation-gated. | +| **Supply-chain integrity as trust signal** | SLSA L2 "an afternoon"; Scorecard weighting; solo-project trust discount | **E9 #27** (packaging) | Confirm cosign-sign + SLSA L2 provenance + SBOM from the first tag; multi-arch + registry-relocatable (China/India). | +| **Cross-cloud enum/cred (incl. China)** | Cast AI's US-only gap; Komodor SaaS excludes air-gap | **E1 #19 / E9 #27 / E12 #30** | Thin per-cloud enum + short-lived token minting (EKS/AKS/GKE + ACK/CCE/TKE); no long-lived keys; air-gap posture. | +| **Cost as read-overlay only** | Cast AI mutates (boundary) | **E13 #31** | Unchanged — OpenCost rollup + GPU columns, read-only; never a metering/optimization engine. | + +**The one new epic to open:** **E14 — Investigation Brain** (the E2 spec already proposes it and +scopes it). It is what turns Sith from "the best OSS multi-cluster viewer" into "the tool that +tells you why, without a black box" — the single most defensible thing in the matrix. + +--- + +## 6. Sources & references + +Primary sources and standards (canonical/stable unless noted). Items marked *(search-surfaced)* +were summarized from a 2026 web search rather than fetched page-by-page — verify before they become +load-bearing in external material. + +**Competitors — clients & platforms** +- k9s repo & multi-cluster issues: · [#1006](https://github.com/derailed/k9s/issues/1006) · [#2730](https://github.com/derailed/k9s/issues/2730) · [#3374](https://github.com/derailed/k9s/issues/3374) · [#1430](https://github.com/derailed/k9s/issues/1430) +- Headlamp: +- kubevious: · · +- Aptakube (multi-cluster): +- Portainer (gaps): · *(search-surfaced)* +- Devtron: +- ArgoCD UI multi-cluster limits: *(search-surfaced)* · + +**Competitors — AI-SRE / auto-triage** *(all search-surfaced unless a repo)* +- k8sgpt: +- HolmesGPT: · CNCF auto-diagnosing post: +- Robusta: +- Botkube: +- Cleric: +- The "hallucination gap": +- Falsification methodology for agentic K8s ops (arXiv): +- Open-source AI-SRE comparison (2026): + +**Standards & guidelines** +- OpenTelemetry K8s semconv: · registry · RC blog +- Kubernetes API conventions: +- client-go exec-plugin auth: +- Open Cluster Management: · cluster-proxy +- Model Context Protocol: · LF donation +- SLSA: · Sigstore/cosign: · supply-chain 2026 overview *(search-surfaced)* +- OpenSSF Scorecard: · CNCF: + +**Internal (prior research & specs, this repo)** +- [`USE-CASE-AND-SHAPE.md`](USE-CASE-AND-SHAPE.md) · [`market-and-form-factor.md`](market-and-form-factor.md) · [`integrations-and-ai-governance.md`](integrations-and-ai-governance.md) · [`identity-connections-security.md`](identity-connections-security.md) +- Specs (PRs #40–#43): Slice-0 foundation · F2.1 7-verb source-adapter contract · E2 four-lens read-federation + Investigation Brain · F11 local fleet UX +- Implementation backlog: master roadmap **#39**; epics **#18–#31**; Phase-L children **#32–#38** diff --git a/docs/research/USE-CASE-AND-SHAPE.md b/docs/research/USE-CASE-AND-SHAPE.md new file mode 100644 index 0000000..650c029 --- /dev/null +++ b/docs/research/USE-CASE-AND-SHAPE.md @@ -0,0 +1,421 @@ +# Sith — what it should be, and the shape it should take + +**Status:** research synthesis (authoritative — reconciles the two research lanes) · **Date:** 2026-07-09 + +This is the single authoritative answer to: *what should Sith be so DevOps / Platform / SRE / +MLOps engineers reach for it by default, and what form should it take?* It reconciles the two +prior research lanes into one document: + +- the **evidence set** (PR #16, `docs/research`): the four-file base — identity/security, + connectors/AI-governance, pains/China/India, and the tool/cost/multi-cloud landscape; +- the **sharper day-0 framing** (PR #15, `docs/research-usecase`): local-first single-binary + CLI/TUI as the day-0 wedge, MCP in v1, and honest sizing of the read wedge. + +Every load-bearing claim carries a primary-source URL here or in a companion file +([market-and-form-factor.md](market-and-form-factor.md), +[identity-connections-security.md](identity-connections-security.md), +[integrations-and-ai-governance.md](integrations-and-ai-governance.md)). `devops-portal` is +treated as lessons-learned, not a template (reviews at +`/Volumes/EXTENDED/checkpoints/devops-portal/review-01..10.md`). + +**Reconcile decision:** this document supersedes both prior `USE-CASE-AND-SHAPE.md` versions. +**Recommend closing PR #15** — its synthesis is fully absorbed here and its sharpest points +(day-0 local-first CLI/TUI, MCP-in-v1, honest wedge-sizing) are adopted. PR #16's other three +evidence files (market, identity, integrations) are unchanged and carried alongside this +reconciled synthesis on the reshape branch. + +--- + +## Executive answer + +**What Sith should be.** Sith should be the tool an engineer reaches for the moment they operate +more than one Kubernetes cluster — and it should earn that reach the way k9s did: `brew install`, +one command, no server, no account, no telemetry, nothing to install on the clusters, value +before any configuration. Day 0, `sith` opens a fast **local fleet view over the kubeconfig +contexts the engineer already has** — a terminal UI and CLI (the k9s-grade wow), with an optional +local web "fleet IDE" (`sith ui`) that is Lens-but-better for people who want a visual surface. +Both read the kubeconfigs on the machine directly; credentials never leave the laptop; there is +no hub, no OCM, no agents. That is the shape every individually-adopted tool actually took, while +the central-control-plane shape it is tempting to build first is the one that stays loved-but-heavy +(self-hosted Backstage runs to 3–12 engineers and 6–12 months, +[Roadie](https://roadie.io/blog/backstage-how-much-does-it-really-cost/)). The multi-tenant hub — +OCM `cluster-proxy`/`managed-serviceaccount`, `Workspace` isolation, the RLS backstop, the +external policy decision point — is all still correct, but it belongs to a **day-N `sith hub` +upgrade** that the same binary becomes when a team outgrows kubeconfig fan-out (clusters behind +NAT/VPCs, shared audit, multi-approver prod), not to the day-0 install an individual adopts. + +**The de facto wedge, and the moat behind it.** The wedge is **"k9s for your whole fleet"** — a +cross-cluster read and correlation view no OSS tool ships ("every cluster where `payments` is +Degraded", "which clusters run image X"). k9s users asked for exactly this twice +([#1006](https://github.com/derailed/k9s/issues/1006), 2021; +[#2730](https://github.com/derailed/k9s/issues/2730), 2024) and both were closed *not planned*; +the server-based OSS options (Clusterpedia, Karpor) are read-only with day-N install friction and +modest adoption; the one tool that ships the experience, Aptakube, is closed-source paid GUI +([aptakube.com/multi-cluster](https://aptakube.com/multi-cluster)). But the wedge is a beachhead, +not the moat — the demand signal is a convenience gap, not an emergency (§5). The **moat is +governed action across the fleet and governed agent access**: typed, signed, approval-gated +cluster operations applied identically to a human and to an AI agent — the position the evidence +finds empty across every incumbent. The single reframing that ties it together: *you earn the +right to govern a fleet by first being the tool the engineer already uses to see it.* Lead with +the read wedge; keep governance as the reason to stay; ship both on one binary, one enforcement +pipeline, one closed vocabulary, whether the driver is the TUI, the CLI, or an agent. + +--- + +## 1. The two wedges (the reframe the plan needs) + +The current charter names one wedge — governed action federation — and puts a local console +*out of scope* ("A single-cluster console / IDE … owned by Headlamp, k9s, Lens", +[SCOPE.md](../SCOPE.md)). That is half right and half a missed on-ramp, and the split matters: + +- **The adoption wedge — the local aggregated fleet client.** How you get ten thousand + individual engineers to install the thing and like it. Won by form and trust, not governance + features. The top barriers to adopting a new OSS tool in 2024 were *too complex to run (46%, + +13 pts)*, *fear of abandonment (46%)*, and *thin docs (45%)* — security scanners ranked far + lower ([CNCF 2024](https://www.cncf.io/reports/cncf-annual-survey-2024/)). The Lens revolt was + about an account wall, a trust break (closed source after a vendor-neutrality promise), + telemetry-by-default, and logs/shell removed from the OSS build — not a missing feature. So the + adoption wedge is a single binary, ten-minute wow, no-account, no-telemetry, permissively + licensed, that answers a fleet-wide question on first run. + +- **The durable wedge — governed action federation with AI as a client.** What makes Sith + defensible and, eventually, what an organization pays to self-host and standardize on. The + research confirms this position is *empty*: OCM ships rollout mechanics but no + approvals/typed-verbs/audit; ACM and Rancher have platform-coupled policy; Kargo gates artifact + promotion only; the AI-SRE incumbents (Komodor, Rancher "Liz", HolmesGPT) stop at + advise/diagnose or go autonomy-first with no approval primitives; the MCP gateways (Kong, Solo + agentgateway, MintMCP, Permit.io) enforce auth and tool-allowlists but have no fleet-aware, + blast-radius-conscious, approval-gated action + ([integrations-and-ai-governance.md](integrations-and-ai-governance.md) §4). The baseline + Kubernetes MCP servers govern with a `--read-only` flag and a denied-resources list and by + default expose generic CRUD plus exec-into-pod + ([containers/kubernetes-mcp-server](https://github.com/containers/kubernetes-mcp-server)). + +The predecessor `devops-portal` had **neither** wedge: no adoption on-ramp (helm-install a +platform, then SSO, then value; it 500'd on a fresh install — review-01, review-05) and no +durable moat (it reverse-proxied and iframed tools that were already better, which review-01 +called *negative value*). Sith wins by holding both: lead with the adoption wedge, defend on the +durable one, and build them on **one shared engine** so the local client and the hub are the same +fleet model rendered several ways. + +## 2. Who feels the pain (personas, with honest caveats) + +Fleet scale is real but is an enterprise/platform-team reality, and the cleanest survey evidence +is thinner than the folklore. The CNCF Annual Surveys — the most neutral source — **do not report +per-organization cluster counts**; their closest signal is that 65% of orgs separate applications +by cluster while namespaces (88%) are the dominant, fastest-growing separation unit +([CNCF 2024](https://www.cncf.io/reports/cncf-annual-survey-2024/)) — which actually cuts against +an over-broad multi-cluster pitch. The ">20 clusters" figure comes from Spectro Cloud, a vendor +that sells multi-cluster management (n=455, ≥250 employees, +[State of Production Kubernetes 2025](https://www.spectrocloud.com/state-of-kubernetes-2025)). It +is corroborated by a **non-vendor** source that matters: the 2025 CNCF Argo CD End User Survey +found **25% of adopters connect Argo CD to more than 20 clusters** +([CNCF, 2025-07-24](https://www.cncf.io/announcements/2025/07/24/cncf-end-user-survey-finds-argo-cd-as-majority-adopted-gitops-solution-for-kubernetes/)). +The persona who owns the pain is the **platform/SRE engineer** — 96% of 500+-employee orgs report +a platform function ([Voice of Kubernetes Experts 2024](https://www.cncf.io/blog/2024/06/06/the-voice-of-kubernetes-experts-report-2024-the-data-trends-driving-the-future-of-the-enterprise/)). + +Crucially, **the pains these people name are complexity, cost, and inconsistency — not +"governance."** "Too complex to run" tied for #1 challenge (46%); over half of orgs admit their +clusters are "snowflakes" and highly manual (Spectro Cloud 2025); the concrete daily version, +articulated by k9s users, is context thrash: the per-cluster-terminal workaround "doesn't scale … +what if someone has 5+ clusters" ([k9s #2730](https://github.com/derailed/k9s/issues/2730)). The +sequencing implication is direct: **the wedge must lead with a pain they feel — seeing and +operating across clusters without thrash — and deliver governance as the thing that makes acting +safe.** The deeper pain table (fleet size, change-driven outages, GPU waste, China/India +constraints) is in [market-and-form-factor.md § Part 1](market-and-form-factor.md#part-1). + +## 3. What makes a tool "de facto" — the CLI/org split + +Defaults split into two adoption patterns, and Sith's current framing is in the harder one. +**Individually-adopted tools win on zero-friction terminal fit:** k9s reads your existing +kubeconfig, needs no server or agent, runs under 50 MB, works over SSH, and covers "~90% of daily +kubectl usage" — its install story *is* the onboarding +([decodeops](https://decodeops.substack.com/p/k9s-the-terminal-ui-that-replaces)). +**Org-adopted platforms are loved-but-heavy because they are frameworks you staff:** self-hosted +Backstage is 3 FTEs in year one, 6–12 months to production, upgrades that require refactoring, and +a persona mismatch (terminal-native engineers vs React/CSS customization) +([Roadie](https://roadie.io/blog/backstage-how-much-does-it-really-cost/)). The middle ground that +individuals still reach for is Argo CD — it won by anchoring one sharp job and being CNCF-neutral +(NPS 79, ~60% of clusters), which is exactly why building Sith's action verbs on Argo CD is +well-founded. + +Two failure modes to design against, both verified. **Trust is fragile for single-vendor OSS:** +when Mirantis closed Lens's source, the community migrated to k9s and re-forked to the +MIT-licensed FreeLens and never returned ([HN](https://news.ycombinator.com/item?id=39811772)). A +solo-team project inherits that discount; the mitigations are a permissive license (Apache-2.0, +chosen), no forced accounts, and ideally a foundation path. **Complexity is itself the adoption +barrier** (the 46% "too complex" finding) — a direct argument for the single-binary, no-server +day-0 shape. + +## 4. Recommended form factor — one binary, three modes, layered adoption + +**One Go binary, one embedded web frontend, three run modes.** This is the shape the evidence +points to and what let Headlamp win the Kubernetes-Dashboard succession +([kubernetes.io, 2026-06-01](https://kubernetes.io/blog/2026/06/01/dashboard-to-headlamp/)): + +| Mode | Command | What it is | Who runs it | Phase | +|---|---|---|---|---| +| **CLI + TUI** | `sith` | k9s-style local fleet view over kubeconfig contexts; `sith get pods -A --all-clusters`, `sith correlate`. Read-only, local, zero config | The individual engineer — the fastest day-0 wow | Day 0 (wedge) | +| **Local web "fleet IDE"** | `sith ui` | The same fleet model as a local web UI on `localhost` — Lens-but-better, kubeconfig-direct, **no account, no telemetry** | The engineer who wants a visual surface | Day 0 (wedge) | +| **Governed MCP server** | `sith serve --mcp` | The same fleet exposed to Claude Code / Codex / Cursor as annotated read tools + typed-intent writes | The AI-native engineer — fastest adoption vector | Day 0 / v1 (§6) | +| **Hub (federated)** | `sith hub` | The same UI served multi-user: OCM minions reach NAT'd/VPC'd clusters, `Workspace` isolation + RLS, external PDP (Ardur), multi-approver prod, shared audit | The platform/SRE team | Day N (moat) | + +Why this shape, with the evidence: + +- **A hub is not required to federate reads.** Aptakube connects to many clusters "as if it was + one big cluster" from the existing kubeconfig with "nothing to install on your clusters" + ([aptakube.com/multi-cluster](https://aptakube.com/multi-cluster)); `kubernetes-mcp-server` + federates "as defined in your kubeconfig files" as a single Go binary; Karpor onboards with a + read-only kubeconfig. Day-0 federation is **client-side kubeconfig fan-out**. +- **But the hub solves the one thing fan-out cannot.** Clusterpedia states it "does not actually + solve … network connectivity in a multi-cluster environment"; Karpor requires the hub to reach + every cluster's API server. Clusters behind NAT / isolated VPCs cannot be reached by a laptop — + which is exactly what OCM `cluster-proxy` + `managed-serviceaccount` solves and Milestone-0 + already proved. So the conclusion is not "drop OCM" — it is **"OCM is the day-N server-mode + transport, not the day-0 dependency."** +- **Single artifact, cache-first render.** k9s (`brew install`, one binary) defines the funnel; + "helm-install a platform then SSO then value" is the cliff `devops-portal` fell off. Render from + a local informer/watch cache in tens of milliseconds, never spinner-first (the Linear local-first + mechanic; the [0.1s/1s/10s limits](https://www.nngroup.com/articles/response-times-3-important-limits/)). +- **Center of gravity is the fleet, not the pod.** The local mode is *not* another single-cluster + console (that slot is taken and correctly out of scope). It is the aggregated, cross-cluster + view Headlamp/k9s/Lens don't center on — with per-pod table stakes (logs, exec, port-forward, + YAML) present because their absence drove the Lens exodus, but not a place to out-feature + Headlamp. Details and the Tauri-not-Electron desktop-shell call are in + [market-and-form-factor.md § Part 2](market-and-form-factor.md#part-2). + +This mirrors what produced defaults, including in MLOps: SkyPilot won as a local-first CLI over +existing credentials, not a server-first control plane +([SkyPilot](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/)). + +## 5. The wedge — "k9s for your whole fleet," honestly sized + +There is genuinely no OSS terminal-native "all my clusters at once" tool, and the demand is +articulated inside the incumbent's own tracker (k9s #1006, #2730 — both closed *not planned*). The +one tool that ships it, including marketed **cross-cluster incident correlation**, is Aptakube's +closed paid GUI. The **10-minute wow:** `brew install sith && sith` → every context rendered as +one fleet, with cross-cluster queries no single-context tool can answer. + +**The honesty caveat, stated plainly.** The k9s demand signal is *modest* — #1006 drew 7 +reactions over five years, #2730 drew 8, and the requester conceded per-context switching "works, +is fast, and is definitely helpful." This is a convenience and efficiency gap, not an unmet +emergency. Two consequences: (1) the read wedge earns adoption but will not by itself make Sith a +*default* — it is the beachhead; (2) the moat must be what the read view *enables* that nothing +else does safely — governed cross-cluster action and governed agent access. The read view is how +you get installed; the governed action + MCP layer is why you stay. + +## 6. AI/MCP as an adoption driver — ship the MCP server in v1, not Phase 3 + +This is the sharpest reprioritization the reconciliation adopts. Agent-driven Kubernetes operation +is a real, fast-growing channel: `containers/kubernetes-mcp-server` (~1.8k stars, ~80k npm +downloads/month) and `Flux159/mcp-server-kubernetes` (~1.1k stars, `claude mcp add kubernetes …`) +have real traction, and 90% of ~5,000 DORA respondents use AI at work +([dora.dev](https://dora.dev/dora-report-2025/)). The anxiety is documented and specific: an +engineer gave Claude a staging cluster and "within minutes it tried to `kubectl exec` … and ran +`kubectl get secret -o yaml`" +([kubectl-ro](https://dev.to/veysi/kubectl-ro-read-only-kubernetes-access-for-ai-agents-and-humans-1okg)); +Replit's agent deleted a production database during an explicit freeze, and the vendor's own +remedy was *structural* — the action "should never be possible" +([Fortune](https://fortune.com/2025/07/23/ai-coding-tool-replit-wiped-database-called-it-a-catastrophic-failure/)). +DORA 2025 finds AI adoption correlates *negatively* with delivery stability absent "robust control +systems." This is the strongest external validation of Sith's thesis: **safety must come from +boundaries the agent cannot cross (typed intents, closed vocabulary, no exec), not instructions it +might ignore.** + +Two constraints from the same evidence. The generic MCP-gateway category is crowded (13+ products +including Kong, Bedrock AgentCore, Docker), so Sith wins only as a **Kubernetes-fleet-operations +gateway with typed action vocabularies**, which none of the generic gateways are +([Obot](https://obot.ai/blog/the-13-best-mcp-gateways-for-enterprise-teams/)). And the prescribed +defense against "shadow MCP" (developers wiring ungoverned servers into Cursor) is to **make the +approved path easier than the unapproved one**. If Sith's governed path is harder than +`npx kubernetes-mcp-server`, engineers route around it. **That is why the MCP server must ship in +v1** — read-only MCP tools ship with the read wedge from the same binary the engineer already +runs; typed-intent writes follow immediately behind the first governed write. MCP is now +vendor-neutral Linux Foundation infrastructure +([donated 2025-12-09](https://blog.modelcontextprotocol.io/posts/2025-12-09-mcp-joins-agentic-ai-foundation/)): +the protocol is settled, the governance on top of it is not. + +## 7. The MLOps dimension — convergent substrate, later verbs + +DevOps and MLOps are converging on Kubernetes ("the de facto operating system for AI"; 66% of orgs +hosting gen-AI use K8s for some inference, +[CNCF 2026-01-20](https://www.cncf.io/announcements/2026/01/20/kubernetes-established-as-the-de-facto-operating-system-for-ai-as-production-use-hits-82-in-2025-cncf-annual-cloud-native-survey/)), +which makes MLOps a natural expansion — but leading with it is the slower path (44% of orgs run no +AI/ML on K8s; only 7% deploy models daily). The scheduling half is already owned: MultiKueue +dispatches batch jobs across clusters, so "which cluster has free GPUs" is answered for submission +([Kueue docs](https://kueue.sigs.k8s.io/docs/concepts/multikueue/)). The real white space is +**utilization and serving** — average GPU utilization near 5%, an organizational/scheduling +problem, and no cross-cluster model-promotion story in KServe. So "kill zombie GPU workloads +across the fleet" and "promote model X across serving clusters" are plausible **later** typed +intents — design the fleet model and verb vocabulary now so they slot in without re-architecture; +lead with DevOps, where the installed base is. GPU cost columns in the fleet model are in scope +early (§8); GPU *action* verbs are a Phase-2+ expansion. + +## 8. Multi-cloud, China/India, cost, connectors (the evidence base) + +The deep material lives in the companion files; the load-bearing conclusions for shape and +priority: + +- **Multi-cloud is two thin layers, not per-cloud products.** The Kubernetes API is uniform across + US and China clouds (Alibaba ACK, Huawei CCE, Tencent TKE are all conformance-certified), so + cluster-*inside* views work day one with no cloud code. Only **enumeration and short-lived + credential minting** differ per cloud (EKS `get-token`, AKS Entra+kubelogin, GKE plugin, + ACK/CCE/TKE) — a thin adapter that stores no long-lived keys + ([market § Part 3 §6](market-and-form-factor.md#part-3)). +- **China/India raise the priority of properties Sith should have anyway.** Air-gap install, + `linux/arm64`+`amd64` multi-arch images, registry-relocatable images, tamper-evident audit + + admin/auditor role separation (maps to MLPS 2.0 Level 3), and a hub that works entirely inside + one network boundary — all mandatory for the China/regulated market and cheap if designed in + early. DPDP/RBI (India) want in-country self-hosting, satisfied by the self-hosted hub + no + phone-home ([market § Part 1 §4–5](market-and-form-factor.md#part-1)). +- **Cost is a read-overlay, not a build.** The fleet rollup is the documented gap (OpenCost is + per-cluster by design; its multi-cluster ask was triaged P3 and closed; Kubecost's unified + multi-cluster view is Enterprise-tier). Deploy/read OpenCost per cluster, aggregate at the hub + into per-workspace/team rollups with GPU columns where DCGM exists. Building a metering engine + re-fights OpenCost/Kubecost/CAST AI ([market § Part 3 §5](market-and-form-factor.md#part-3)). +- **Connectors are three fixed kinds, not an open ecosystem.** Grafana (out-of-process gRPC, + SDK-first) and Terraform (versioned, minor-additive protocol) scaled; Backstage (in-process, + unversioned, 250+ plugins, breaking changes in minor releases) drowned. Sith's framework is + out-of-process gRPC, SDK-first, **one canonical connector per tool**, three kinds only: read + adapter / brokered read-through (deep-link, never re-skin) / typed-action adapter. The day-1 six + that feed the model cheaply or host the first write are **Argo CD, Flux, Helm, Prometheus, Loki, + GitHub** ([integrations-and-ai-governance.md](integrations-and-ai-governance.md) §1–2). + +## 9. Competitive white-space map + +Rows are jobs an engineer wants done; "—" means the tool does not do that job. + +| Job / pain | k9s | Aptakube | Headlamp / Lens | Clusterpedia / Karpor | Komodor (SaaS) | OCM (substrate) | kubernetes-mcp-server | **Sith (proposed)** | +|---|---|---|---|---|---|---|---|---| +| Single-cluster ops UI/TUI | ✅ default | GUI | ✅ GUI | — | ✅ | — | — | ✅ (reuse pattern) | +| **All-clusters-at-once view (OSS, local)** | ❌ (asked, unbuilt) | ✅ closed/paid GUI | ❌ | server-install, read-only | ✅ SaaS | — | via kubeconfig, no UI | ✅ **wedge** | +| Cross-cluster correlation query | — | ✅ GUI | — | search only | ✅ | — | — | ✅ | +| Local-first, no server/agent | ✅ | ✅ | ✅ desktop | ❌ hub | ❌ SaaS agent | ❌ hub+agents | ✅ | ✅ (day 0) | +| Cross-VPC/NAT reach | — | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ (day-N hub, via OCM) | +| Governed typed actions (no exec) | ❌ (exec yes) | ❌ | ❌ | ❌ | proprietary | — | coarse on/off | ✅ **moat** | +| Governed MCP server for agents | — | — | — | — | MCP client-side | — | coarse flags | ✅ **moat** | +| Multi-tenant workspace + audit + PDP | — | — | — | — | ✅ SaaS | RBAC only | — | ✅ (day-N) | +| OSS + vendor-neutral | ✅ | ❌ | mixed | ✅ | ❌ | ✅ | ✅ | ✅ | + +The empty column no incumbent fills: **OSS + local-first + all-clusters + governed typed actions + +governed MCP, in one binary that scales to a shared hub.** Sith's defensibility is the +combination, not any single cell. + +## 10. The ruthlessly prioritized roadmap + +Feature sprawl killed `devops-portal` (12 providers, 92 routes, ~7/10 pillars pure pass-through — +review-01). Every capability the owner named is placed in exactly one bucket, and "not now" is as +important as the wedge. A capability earns a higher bucket only if it serves a wedge. + +**Wedge (build first — reason to exist + on-ramp):** +- Local fleet client — `sith` CLI/TUI (k9s-style, the fastest wow) **and** `sith ui` (local web + fleet IDE, "Lens but better"), kubeconfig-direct, no account/telemetry, cache-first, cmd-K/`:` + fleet search, logs/exec/port-forward/YAML in core. +- **Source-agnostic** read federation + normalized fleet model + cross-cluster correlation — the + shared engine behind every mode; source is local kubeconfig (day-0) *or* OCM spoke (day-N). +- **Governed MCP read server** from the same binary (`sith serve --mcp`) — annotated read tools in + v1 (the shadow-MCP argument makes this a hard requirement, §6). +- Minions (outbound OCM agents) + multi-auth (kubeconfig for local; API key / JWT / OIDC and + short-lived cloud IAM for the hub). +- Governance spine, day one: `Workspace` tenancy, signed-token authn (never header trust), + least-privilege RBAC, forced Postgres RLS backstop, audit-log + decision-ledger. +- First governed typed write — `gitops.open-pr` through the Ardur PDP; MCP write tool for it lands + right behind it. +- No-god-key custody: no central admin kubeconfig; KMS-envelope per-tenant DEKs (hub), OS keychain + (local); cosign-signed releases + SLSA L2 provenance + SBOM from the first tag. + +**Fast-follow (right after the wedge proves out):** +- Policy federation — waves/canary, environment gates, multi-approver for prod, partial-failure / + auto-rollback, abstention; the live-mutation verbs (`argocd.sync`, `rollout.promote|abort`, + `deployment.scale|restart`) behind all of it. +- Connector framework — out-of-process gRPC, SDK-first, three fixed kinds — generalizing the day-1 + six. +- Cost read-overlay — OpenCost rollup + GPU columns; not a metering engine. +- Multi-cloud enumeration + short-lived token minting incl. ACK/CCE/TKE; OpenShift conformant-API + coverage. +- Air-gap / multi-arch / registry-relocatable packaging (Zarf-style, no phone-home); Tauri desktop + shell. + +**Later (real, but not until the above lands):** +- Long-tail read connectors (Datadog, Splunk, Elastic/OpenSearch/Kibana, Terraform/OpenTofu + state-drift); ITSM typed actions (Jira/Zendesk/ServiceNow change tickets). +- Sith as an MCP *client* (calling kagent, Grafana MCP, GitHub MCP); governing LangChain/LangGraph + agents (they connect as MCP clients of Sith, get a scoped identity, ride the same PEP — Sith + governs, it does not orchestrate). +- MLOps typed verbs (kill zombie GPU jobs, promote model across serving clusters); OpenShift-specific + views (Routes/SCC). + +**Explicitly not now (say no, loudly — the anti-sprawl contract):** +- Re-skinning / proxying tool UIs (the `devops-portal` iframe-Grafana trap) · a telemetry lake / + metrics store · a metering / cost-optimization engine · an agent-orchestration framework (that is + LangGraph/kagent) · a developer portal / IDP / catalog / scorecards / DORA · a GitOps reconciler · + a multi-cluster scheduler · Fluentd/Fluent-bit as data sources (read the sinks) · Kustomize/Helm + as action targets in v1 · `exec` / free-form apply / Secret / RBAC mutation (permanently + inexpressible) · running SPIRE (support SPIFFE IDs/mTLS, don't force the platform). + +## 11. Carry / discard / net-new vs `devops-portal` + +| Carry (the good bones) | Discard (the failure modes) | Net-new (what the research says Sith needs) | +|---|---|---| +| Action/exec broker service-layer (clean 1:1 tool→service map, review-08) — **redesigned as the PEP** with a closed vocabulary, no shell | Shared central admin kubeconfig / inbound-god-kubeconfig — replaced by outbound OCM minions + scoped MSA tokens + no central admin cred | Local aggregated fleet client (CLI/TUI + web) as the adoption wedge (empty OSS slot) | +| Per-org encrypted credential vault (AES-256-GCM key-ring, review-08) — **re-architected** as KMS-envelope per-tenant DEKs | Single god key (`TOKEN_ENCRYPTION_KEY` decrypts every tenant) | Cross-cluster correlation as a first-class query in a local tool (k9s won't build it; only a closed GUI ships it) | +| RBAC + audit spine (review-08) — kept, **hardened** with signed-token authz + a separate decision-ledger | Dead/inert RLS + `x-user-role` header-trust IDOR — replaced by FORCE RLS (non-owner role) + signed-token-only authz | Typed-intent action model + signed dispatch + per-minion local allowlist re-validation (two independent blast-radius bounds) | +| Governed AI/MCP ambition — review-10's one salvageable idea — becomes the **core, done right** (real MCP server in v1, elicitation gates, AI-as-client) | All-heavy monolith (~48k LOC, 92 routes; iframes better tools) — replaced by one narrow Go binary + local surfaces, no re-skinning | Policy federation — waves / multi-approver / abstention (novel, empty) | +| `Workspace`-over-clusters tenancy — kept as the isolation anchor with a **real** RLS backstop (day-N hub) | Feature sprawl (12 providers, ~7/10 pass-through) — replaced by the closed vocabulary + three connector kinds + a hard scope gate | Governed MCP server as a Kubernetes-fleet-ops gateway (generic gateways aren't fleet-aware) | +| OCM `cluster-proxy` + `managed-serviceaccount` — **re-scope to day-N server mode** (solves cross-VPC reach fan-out can't) | Broken onboarding (platform-install cliff; 500s on fresh install) — replaced by `brew install && sith`, ten-minute wow | Air-gap / multi-arch / registry-relocatable distribution for China/regulated | +| Typed-intent closed vocabulary, no exec — **carry, strengthen** (boundaries-not-instructions is the verified lesson) | Features that never ran (dead write path, mock MCP page, qwen2.5:3b) — replaced by falsification-first, ship-what's-verified | Cost read-overlay with fleet rollup + GPU columns (empty in OSS, paywalled commercially) | +| | "Central control plane + web UI" as the *day-0* shape — **change** to local-first day-0, hub day-N | MLOps typed verbs (kill zombie GPU jobs, promote model across clusters) — build later | + +## 12. Concrete changes to the plan + +The governance thesis is *validated* by the evidence (annotations are hints, enforcement must be +server-side, no incumbent occupies governed fleet action). The change is to add the adoption wedge, +invert the day-0/day-N shape, pull the MCP read server into v1, and hang the named capabilities off +the wedge without diluting the anti-drift contract. + +- **CHARTER** — reframe from one wedge to **two** (adoption = local client; durable = governed + federation); add the **individual operator** as the top-of-funnel user; add an adoption success + criterion (`brew install` → populated cross-cluster answer in <10 min, offline, nothing leaves + the machine); state the two share one engine. +- **SCOPE** (the key edit) — refine "single-cluster console — out of scope" to "*another* + single-cluster console" and add **in-scope** "aggregated multi-cluster local fleet client (the + adoption on-ramp), distinguished by fleet aggregation/correlation, not per-pod parity." Make the + §10 "not now" list contractual. +- **ARCHITECTURE** — add the three run-modes on one Go binary + embedded frontend; make read + federation **source-abstract** (local kubeconfig *or* OCM spoke); add the four-mode identity + model, the connector framework (three kinds), the cost overlay; note cosign/SLSA/SBOM and + SPIFFE-without-SPIRE. +- **ROADMAP** — invert to local-first day-0; sequence **P1 local read + correlation → P1 governed + MCP read tools (same binary) → P2 first governed write (`gitops.open-pr`) + its MCP write tool → + P3 the hub (OCM cross-VPC reach, multi-tenant isolation, multi-approver, wave/canary)**. Keep the + falsification gate; require multi-arch images from the first release. +- **EPICS** — amend E2 to treat local kubeconfig contexts as a first-class read source; pull the + MCP *read* tools (E7) forward into P1; add **E11 (local fleet client)**, **E12 (connector + framework)**, **E13 (cost overlay)**; amend E9 (multi-arch/air-gap/signing) and E8 (the console + is the one frontend served by `sith ui` and `sith hub`). + +## 13. Evidence grading and honesty notes + +- **Verified 3-0 (verbatim vs primary source):** the identity/security claims (exec-plugin auth, + no cert revocation, short-lived-token guidance), the connector-framework claims (Grafana + out-of-process, Terraform protocol versioning, Argo/Flux/Helm/Prom/Loki/Grafana surfaces), the + MCP annotation claims; plus CNCF 2024 namespace-vs-cluster figures and "too complex to run" (46%). + Full votes in the companion files. +- **Refuted / corrected:** the "five or more clouds" phrasing was a misread — the source says + "five-plus clouds **and environments**"; the >20-cluster core stands and is corroborated by the + non-vendor CNCF Argo CD survey (25% >20 clusters). +- **Primary-source quotes not independently re-verified (a verification pass was cut short by a + session limit):** the k9s issue histories, Aptakube capabilities, Clusterpedia/Karpor + architecture, MCP-server adoption numbers, kubectl-ro, Replit, DORA 2025, Backstage TCO, the + China/India source set. High-confidence, direct quotes, not triple-checked. +- **Vendor-sponsored (discount for framing bias):** Spectro Cloud (sells multi-cluster mgmt), CAST + AI, Komodor, Roadie (Backstage cost testimony is against interest, so more credible). Fleet-scale + premise independently corroborated by the CNCF Argo CD survey. + +--- + +**The one-line test for every future feature request:** does it serve the adoption wedge (get an +individual to install and love the local fleet client) or the durable wedge (governed action +across the fleet, humans and agents alike)? If neither, it belongs in §10's "not now" — no matter +how useful it sounds. That question, applied ruthlessly, is the difference between Sith and the +predecessor. diff --git a/docs/research/identity-connections-security.md b/docs/research/identity-connections-security.md new file mode 100644 index 0000000..a1a8db7 --- /dev/null +++ b/docs/research/identity-connections-security.md @@ -0,0 +1,84 @@ +# Connection/identity modes and security architecture (workstreams D + J) + +**Date:** 2026-07-09 · **Method:** deep-research fan-out; claims below carry a primary URL. Items marked **[3-0]** passed adversarial 3-vote verification; items marked **[fetched]** were pulled from the primary source but the verification vote was rate-limited (the source and quote are real; treat the framing as single-reviewer). Nothing here is asserted without a URL. + +Answers: how Sith should connect and broker identity across its four connection modes, and what the security bar actually is for a tool that touches many clusters and holds many secret types. + +--- + +## 1. Kubeconfig reality — why "upload your kubeconfig" is the wrong primitive + +The predecessor's central-kubeconfig honeypot wasn't just risky, it was **technically broken for modern clusters**: + +- **[3-0]** kubectl/client-go authenticate to clusters via **exec credential plugins** — the kubeconfig instructs the client to run an external command locally to obtain credentials; this is the mechanism behind `aws eks get-token`, `kubelogin`, and `gke-gcloud-auth-plugin` ([k8s auth docs](https://kubernetes.io/docs/reference/access-authn-authz/authentication/)). A cloud kubeconfig **contains no usable credential by itself** — it points at a helper binary and the user's local cloud session. +- **[3-0]** kubectl **removed built-in AKS/GKE auth**; "Earlier versions of kubectl included built-in support for authenticating to AKS and GKE, but this is no longer present" ([same](https://kubernetes.io/docs/reference/access-authn-authz/authentication/)). From v1.26, GCP provider auth was removed from OSS kubectl; GKE now **requires** the external `gke-gcloud-auth-plugin` ([GKE auth changes](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke)). +- **Consequence:** uploading a cloud kubeconfig to a server-side tool **does not transfer working auth** — the server lacks the plugin and the user's cloud session. So the honeypot design is both dangerous *and* non-functional for EKS/AKS/GKE. This is decisive evidence for **local mode keeping kubeconfigs on the user's machine** and **federated mode using in-cluster agents**, not a central credential store. +- **[3-0]** Kubernetes (through v1.36) has **no X.509 client-cert revocation** — an issued cert is valid until expiry, so a leaked admin kubeconfig with an embedded cert is an **irrevocable standing credential** ([k8s auth docs](https://kubernetes.io/docs/reference/access-authn-authz/authentication/)). +- **[3-0]** OIDC id_tokens as bearer tokens "can't be revoked… so [they] should be short-lived (only a few minutes)" — upstream endorsement of the short-lived model ([same](https://kubernetes.io/docs/reference/access-authn-authz/authentication/)). +- **[fetched]** The leak evidence is real: Aqua Nautilus's 2023 GitHub scan found 438 public records with base64 K8s registry secrets, **~46% still valid** ([Aqua](https://www.aquasec.com/blog/the-ticking-supply-chain-attack-bomb-of-exposed-kubernetes-secrets/)); exposed GCP/AWS tokens in the same repos were **already expired** — short-lived creds converted leaks into non-events. Microsoft's 38TB exposure came from a single over-privileged SAS token valid **~3 years** before remediation ([Wiz](https://www.wiz.io/blog/38-terabytes-of-private-data-accidentally-exposed-by-microsoft-ai-researchers)). + +## 2. The outbound-agent pattern is the industry consensus (Sith's minion mode is not exotic) + +Every serious multi-cluster broker uses an in-cluster agent that dials **out**; none require inbound reach: + +| System | Directionality (primary source) | +|---|---| +| **OCM** klusterlet + cluster-proxy | Outbound-only; hub needs no inbound access — reproduced hands-on in [M0](../experiments/M0-ocm-falsification.md) | +| **Teleport** | **[fetched]** Agents connect via an **outbound reverse tunnel** to the Proxy; sit behind NAT/firewall with no inbound. K8s Service runs **as a pod using its own service-account** — no kubeconfig exported ([Teleport agents](https://goteleport.com/docs/reference/architecture/agents/)) | +| **Azure Arc** | **[fetched]** "No inbound ports… agents communicate with Azure exclusively via outbound connections"; `clusterconnect-agent` brokers apiserver reach over an agent-initiated tunnel ([Arc agent overview](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/conceptual-agent-overview)) | +| **GKE Connect** | **[fetched]** Connect Agent "initiates an outbound connection to Google… works through NATs, egress proxies, and firewalls" ([Connect Agent](https://cloud.google.com/kubernetes-engine/fleet-management/docs/connect-agent)) | +| **Rancher** | **[fetched]** `cattle-cluster-agent` "opens a tunnel out to a cluster controller inside the Rancher server" ([Rancher arch](https://ranchermanager.docs.rancher.com/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters)) | +| **HashiCorp Boundary** | **[fetched]** Multi-hop: the egress worker "initiates outbound connections… for networks that forbid inbound traffic" ([Boundary multi-hop](https://developer.hashicorp.com/boundary/docs/workers/multi-hop)) | + +Two design corollaries the sources hand us. (a) **"Outbound-only agent" is table stakes, not differentiation** — every incumbent has it (and Karmada has pull mode too). Sith's differentiation must be the governance *above* the transport, not the transport. (b) Both Rancher and Arc broker access through a **central auth proxy/chokepoint** — validating "one governed entry point in front of every cluster API call", which is exactly Sith's PEP. + +### Short-lived credentials are settled consensus + +- **[fetched]** SPIFFE/SPIRE **graduated from CNCF (2022-09-20)** with named production adopters (Bloomberg, ByteDance, Netflix, Pinterest, Uber…); design goal is "removing the need for shared secrets" via attested, auto-rotated X.509/JWT SVIDs ([CNCF SPIFFE graduation](https://www.cncf.io/announcements/2022/09/20/spiffe-and-spire-projects-graduate-from-cloud-native-computing-foundation-incubator/)). +- **[fetched]** **Pinniped** is direct prior art for "one upstream identity, many clusters without distributing static per-cluster creds": its Supervisor federates identity, its Concierge **exchanges a token for a short-lived mTLS client cert** per target cluster; on managed clouds (no signing-key access) it falls back to an **in-cluster impersonation proxy** ([Pinniped arch](https://pinniped.dev/docs/background/architecture/)) — evidence a multi-cluster tool **cannot rely on one auth mechanism across cluster types**. +- Vault Transit and cloud STS AssumeRole round out the brokered-credential toolbox ([Vault Transit](https://developer.hashicorp.com/vault/docs/secrets/transit)). + +### The market lesson: Infra (infrahq) + +- **[fetched]** Infra (YC W21) built exactly this category — a broker for K8s access via connectors + access keys + IdP integration — and is **dormant**: last release v0.21.0 (2023-01-25), no release in ~3.5 years; founders left to build Ollama ([Infra releases](https://github.com/infrahq/infra/releases)). **Lesson:** a standalone "cluster access broker" is a thin, hard-to-monetize wedge on its own. Sith must not position as "another access broker" — access brokering is a *property* of the governed action layer, not the product. (Corroborated by the Lens/Kubecost/Rancher consolidations: point tools in this space get absorbed or stall.) +- `kube-oidc-proxy` (a commonly-cited OIDC-for-managed-clusters tool) was **archived by jetstack 2024-05-17** ([repo](https://github.com/jetstack/kube-oidc-proxy)) — another reason not to depend on a single OSS auth shim. + +## 3. Recommended multi-mode connection + identity model + +| Mode | How it connects | Identity/custody | Security property | +|---|---|---|---| +| **Local (direct)** | Reads the user's existing kubeconfig contexts on the machine; exec plugins run locally exactly as kubectl does | **Kubeconfigs never leave the machine**; any Sith-held secret (rare in local mode) goes in the **OS keychain** | Same trust boundary as kubectl. No server, no upload, no honeypot. Matches how Docker/gh CLI store local creds (§4) | +| **Federated (minion)** | OCM klusterlet + cluster-proxy, outbound-only; hub reaches cluster-local services via the reverse tunnel | **Scoped `managed-serviceaccount` projected token**; hub holds **no admin kubeconfig** (M0-proven); action execution uses an Ardur-brokered short-lived identity re-validated at the spoke | Reach decoupled from privilege; ceiling below the human's; per-spoke local allowlist | +| **Cloud IAM** | Thin per-cloud adapter enumerates clusters and **mints short-lived tokens** (EKS get-token / AKS Entra+kubelogin / GKE plugin); never stores long-lived cloud keys | STS/AssumeRole-style short-lived creds; nothing standing at rest | Leak = non-event (creds expire); matches the Aqua/Wiz evidence | +| **API key / JWT / OIDC** | For tool integrations (Argo/Grafana/etc.) and machine callers of Sith itself | KMS-envelope, per-tenant DEKs in federated mode; keychain in local mode | Bounded blast radius per tenant; see §5 | + +**One rule across all four:** Sith authenticates *from signed token claims, never spoofable headers* (the predecessor's IDOR), and the credential ceiling for any agent/AI actor is strictly below the human's. + +## 4. Local-mode credential custody — the keychain decision + +- **[fetched]** Docker ships official credential helpers (`osxkeychain`, `wincred`, `secretservice`, `pass`) precisely because the prior default — base64 in `~/.docker/config.json` — is trivially decodable plaintext ([Docker credential helpers](https://github.com/docker/docker-credential-helpers)). This is the established desktop-tool norm. +- **[fetched]** GitHub CLI moved to system-keyring by default (Feb 2023) but has a **silent plaintext fallback** when no keyring is available — closed by warning rather than fail-closed ([gh#7570 / PR #7781](https://github.com/cli/cli/issues/7570)). **Design lesson for Sith:** keychain-first, but the fallback must **fail loudly or encrypt-at-rest**, never silently write plaintext. + +## 5. Server-side custody + tenancy (federated mode) + +- **Envelope encryption, per-tenant DEKs wrapped by a KMS KEK** — one KMS key per tenant, encryption-context as a cryptographic tenant boundary ([AWS SaaS envelope pattern](https://docs.aws.amazon.com/prescriptive-guidance/latest/saas-multitenant-managed-policies/data-encryption.html)); Vault Transit as the non-cloud-KMS alternative (Vault holds keys, never the data — [Transit docs](https://developer.hashicorp.com/vault/docs/secrets/transit)). This is the exact fix for the predecessor's single env master key. +- **Postgres RLS** is the validated pooled-tenancy backstop, with the well-documented caveat that it only isolates if the app connects as a **non-owner, non-BYPASSRLS** role and `FORCE ROW LEVEL SECURITY` is set ([AWS SaaS Factory RLS](https://aws.amazon.com/blogs/database/multi-tenant-data-isolation-with-postgresql-row-level-security/), production report: [Nile](https://www.thenile.dev/blog/multi-tenant-rls)). Operational hazards to engineer around: session-variable RLS vs PgBouncer pooling, and thread-local context leaking across reused connections (real reported bug). Sith's plan already has RLS from day one — the sources confirm the exact pitfalls to avoid. + +## 6. Supply-chain + audit checklist, ranked by what evaluators actually check + +Evaluators of an OSS infra tool run automated scorecards before humans read code. Ranked by what those tools weight: + +1. **Signed releases (sigstore/cosign) + CI hygiene** — OpenSSF Scorecard rates release signing **High** risk and dangerous GitHub Actions workflows **Critical**; CNCF "highly recommends" Scorecard and wires it into CLOMonitor ([OpenSSF Scorecard](https://github.com/ossf/scorecard), [CNCF security guidance](https://contribute.cncf.io/maintainers/community/compliance/)). Notably, **Scorecard has no SBOM check** — signing and CI hygiene outrank SBOM in practice. +2. **SLSA build provenance** — a maturity ladder projects should climb (L1 provenance → L3 tamper-resistant); "should", not a hard CNCF mandate ([slsa.dev](https://slsa.dev)). +3. **SBOM (SPDX/CycloneDX), ideally signed** — expected of producers; consumers ask for it ([CNCF supply-chain paper](https://github.com/cncf/tag-security/blob/main/community/resources/software-supply-chain-security/secure-supply-chain-guidance/)). +4. **Append-only, tamper-evident audit log** — SOC 2 auditors expect WORM or hash-chained logs; they judge the **evidence artifacts you can produce**, not a config toggle; each entry needs actor/action/timestamp/resource; maps to CC6.1/CC6.3/CC7.2 ([Bytebase SOC2 report](https://www.bytebase.com/blog/soc2-audit-logging/)). This validates Sith's separate **audit-log (what-happened) + decision-ledger (why-allowed)**. +5. **Third-party security audit** (Cure53-style) — expected for security-adjacent tooling (SPIFFE required it for graduation). + +### Signed-action precedent and EU AI Act + +- **Signed intents have direct precedent:** Teleport signs short-lived certs encoding client identity + routing; in-toto attestations are the CNCF-endorsed "signed records of actions" pattern for a verifiable action ledger, with Sigstore keyless signing tying signatures to identity not long-lived keys ([CNCF supply-chain paper](https://github.com/cncf/tag-security/blob/main/community/resources/software-supply-chain-security/secure-supply-chain-guidance/)). Sith's signed typed intents sit squarely in this lineage. +- **EU AI Act Article 12** record-keeping is **scoped to high-risk AI systems only** ([Article 12 text](https://artificialintelligenceact.eu/article/12/)). A K8s ops tool with AI features is bound **only if classified high-risk**. So it's not a blanket obligation — but the granularity regulators expect (per-action, timestamped, human-attributed, tamper-evident) is exactly the ledger Sith already plans, which is why "build the ledger regardless" is the right call: it satisfies SOC 2 now and AI-Act-high-risk later at no extra design cost. + +## 7. China/India compliance implications for architecture + +The China/India constraints in [market-and-form-factor.md § Part 1](market-and-form-factor.md#part-1) map onto this security design cleanly: MLPS 2.0 L3 wants **admin/auditor role separation** (Sith's reader/operator/approver/admin + separate audit vs decision ledgers), **tamper-resistant backed-up audit records** (append-only/hash-chained log, item 4 above), and **no root SSH / agent-mediated access** (the outbound-agent + brokered-identity model *is* that). DPDP/RBI want in-country self-hosting — satisfied by the self-hosted hub + no phone-home. None of this requires China-specific code; it requires designing the audit/role/custody spine correctly once. diff --git a/docs/research/integrations-and-ai-governance.md b/docs/research/integrations-and-ai-governance.md new file mode 100644 index 0000000..4363030 --- /dev/null +++ b/docs/research/integrations-and-ai-governance.md @@ -0,0 +1,82 @@ +# Integration framework and AI-agent governance (workstreams H + I) + +**Date:** 2026-07-09 · **Method:** deep-research fan-out; **[3-0]** = adversarially verified, **[fetched]** = pulled from the primary source, vote rate-limited. Every load-bearing claim carries a URL. + +Answers: how to design the connector framework, the per-tool integration mechanism and priority, and whether "governed MCP server for fleet actions" is a real, empty position. + +--- + +## 1. Connector-framework design principles (what scaled, what drowned) + +**Grafana's model scaled — copy it.** +- **[3-0]** Grafana runs backend plugins **out-of-process as subprocesses over gRPC** (HashiCorp go-plugin), so "a panic in a plugin doesn't panic the server" ([Grafana backend plugins](https://grafana.com/developers/plugin-tools/key-concepts/backend-plugins/)). +- **[3-0]** The plugin contract is a **small fixed set of typed capabilities** (query, resources, health, metrics, stream) — narrow schema, not arbitrary host access. +- **[3-0]** It's **SDK-first**: authors code against `grafana-plugin-sdk-go`, which hides the wire protocol — the mechanism that let Grafana evolve the protocol without breaking the ecosystem ([grafana#19667](https://github.com/grafana/grafana/issues/19667)). + +**Terraform confirms the versioning discipline.** +- **[3-0]** The provider protocol is a **versioned, typed gRPC/protobuf interface**; **major versions delineate compatibility, minor versions are strictly additive (non-breaking)** ([Terraform plugin protocol](https://developer.hashicorp.com/terraform/plugin/terraform-plugin-protocol)). + +**Backstage is the cautionary tale — in-process, unversioned, drowned in maintenance.** +- **[fetched]** A BackstageCon EU 2026 maintainer panel (Red Hat/DoorDash/OP Financial/Vodafone Ziggo): the marketplace has **250+ plugins, many unmaintained**; **breaking changes ship within minor releases despite semver** (1.48→1.49), stranding adopters several releases behind; the React Router 6→7 migration was especially costly; adopters must write **custom React plugins**, taking on frontend dev on top of their real job ([panel writeup](https://tldrecap.tech/posts/2026/backstagecon-europe/backstage-plugin-ecosystem-sustainability/)). Proposed fixes: **bind backend plugins to OpenAPI schemas** for stronger break detection; **one canonical plugin per target** (not many overlapping); **quality-tier/ownership signaling**. + +**Transferable principles for Sith's connector framework:** +1. **Out-of-process, typed, versioned.** gRPC/protobuf connectors with a stable minor-additive contract (Grafana + Terraform). A crashing connector must not take the hub down. +2. **SDK-first, protocol hidden.** Authors implement a narrow interface; Sith owns the wire format and can evolve it. +3. **Three fixed connector kinds, not open-ended:** **read adapter** (pull normalized facts into the fleet model), **brokered read-through** (proxy to a tool's own UI/API via cluster-proxy — no re-skinning), and **typed-action adapter** (map a closed verb to the tool's API). Everything is one of these three; nothing gets arbitrary host access. +4. **One canonical connector per tool.** Backstage's redundancy-and-abandonment failure is the thing to prevent structurally. +5. **Action semantics belong in the schema.** Flux's HelmRelease encodes drift-detection and remediation (retry/rollback/uninstall) as **typed fields** ([Flux HelmRelease](https://fluxcd.io/flux/components/helm/helmreleases/)) — proof that guardrails can live in the connector contract, not imperative glue. + +## 2. Per-tool integration surface, verified + +| Tool | Mechanism (verified) | Auth | Classification | +|---|---|---|---| +| **Argo CD** | **[3-0]** REST API w/ Swagger at `/swagger-ui` ([API docs](https://argo-cd.readthedocs.io/en/latest/developer-guide/api-docs/)) | **[3-0]** Bearer JWT via `/api/v1/session` | Read adapter **+ typed actions** (`argocd.sync\|rollback`) — day 1 | +| **Flux** | **[3-0]** **CRD-only** (`helm.toolkit.fluxcd.io/v2`); integrate by patching CRs + reading `.status` ([Flux](https://fluxcd.io/flux/components/helm/helmreleases/)) | Cluster RBAC | Read adapter (CRD), typed action later | +| **Helm** | **[3-0]** No server API — release state is **Kubernetes Secrets** in the release namespace ([Helm advanced](https://helm.sh/docs/topics/advanced/)) | Cluster RBAC | Read adapter via K8s API — day 1, free | +| **Prometheus** | **[3-0]** Stable `/api/v1`, **non-breaking additions only**; destructive TSDB ops segregated under `/admin` and **disabled by default** ([Prom API](https://prometheus.io/docs/prometheus/latest/querying/api/)) | proxy/none | Read adapter — day 1, low-churn | +| **Loki** | **[3-0]** Versioned `/loki/api/v1/query[_range]`, LogQL; **no built-in authz** — front it yourself ([Loki API](https://grafana.com/docs/loki/latest/reference/loki-http-api/)) | external | Read adapter | +| **Grafana** | **[3-0]** HTTP API w/ **service-account tokens** (replaced API keys) ([Grafana SA](https://grafana.com/docs/grafana/latest/administration/service-accounts/)) | SA token | Brokered read-through (link/deep-link; don't re-skin) | +| **GitHub/GitLab** | REST/GraphQL; GitHub App > PAT for scoping | App/OIDC | Typed action host for `gitops.open-pr` — day 1 (the first write) | +| **Terraform/OpenTofu** | State + HCP/Cloud API; providers via the protocol above | tokens | Read adapter (state/drift) later; never a Terraform runner | +| **Datadog / Splunk / Elastic/OpenSearch** | REST/query APIs | API/app keys | Read adapter, later; Datadog cost pain is a *pull* driver, not a reason to embed | +| **Fluentd / Fluent Bit** | **Config-only, no query API** | — | **Skip** as a data source (they ship logs *to* Loki/Elastic; read those instead) | +| **Jira / Zendesk / ServiceNow** | REST APIs | tokens | Typed action (open a change ticket for a fleet action) — later; ITSM change-linkage is real demand but not wedge | + +**Priority rule (from the wedge):** day-1 integrations are the ones that (a) feed the fleet model cheaply via the K8s API or a stable REST endpoint, or (b) host the first governed write. That's **Argo CD, Flux, Helm, Prometheus, Loki, and GitHub** — everything else is demand-ranked fast-follow, and Fluentd/Fluent Bit are skipped as sources. + +## 3. MCP protocol state (2026) — the protocol gives vocabulary, not enforcement + +- **[3-0]** Tool annotations (`readOnlyHint`/`destructiveHint`/`idempotentHint`/`openWorldHint`) shipped in the **2025-03-26** revision ([MCP annotations post](https://blog.modelcontextprotocol.io/posts/2026-03-16-tool-annotations/)). +- **[3-0]** The spec is explicit that **annotations are hints, untrusted unless from a trusted server** — a *risk vocabulary, not a security control*. +- **[2-0]** MCP maintainers state safety guarantees **must be enforced by deterministic controls outside the protocol** (network controls, sandboxing) — official acknowledgment that **governance is out of scope for the protocol**, left to gateways/governed servers. +- **[fetched]** Timeline: **2025-06-18** added **elicitation** (server-initiated mid-flow user input — the primitive HITL approval builds on) and formalized MCP servers as **OAuth Resource Servers**; **2025-11-25** matured the OAuth/OIDC-discovery authz stack and extended elicitation ([2025-06-18 changelog](https://modelcontextprotocol.io/specification/2025-06-18/changelog), [2025-11-25 changelog](https://modelcontextprotocol.io/specification/2025-11-25/changelog)). **Governance-relevant changes stop at auth + scope consent** — no policy/approval/blast-radius primitives. +- **[fetched]** **2025-12-09:** Anthropic donated MCP to the **Agentic AI Foundation**, a Linux Foundation directed fund (co-founded with Block and OpenAI; supported by Google, Microsoft, AWS, Cloudflare, Bloomberg), putting the protocol under "the same neutral stewardship that supports Kubernetes" ([MCP blog](https://blog.modelcontextprotocol.io/posts/2025-12-09-mcp-joins-agentic-ai-foundation/), [LF press release](https://www.linuxfoundation.org/press/linux-foundation-announces-the-formation-of-the-agentic-ai-foundation)). MCP is now vendor-neutral infrastructure — which *strengthens* the case for a neutral OSS governance layer on top: the protocol is a shared standard, the governance of what agents may do through it is still unbuilt. + +**Implication:** Sith's charter is right that MCP annotations are hints and enforcement must be server-side. The elicitation primitive is the correct native shape for the approval gate. The protocol will not govern for you — the governed MCP *server* is the product. + +## 4. Who governs AI agents on clusters today — the position is open + +**The demand is real and incident-backed:** +- **[fetched]** July 2025: **Replit's AI agent deleted the production database** of SaaStr's founder during "vibe coding", **despite explicit instructions not to change anything**; Replit's tooling then wrongly claimed the deletion was unrecoverable ([The Register](https://www.theregister.com/2025/07/21/replit_saastr_vibe_coding_incident/)). The canonical "agent damaged prod" incident. +- **[fetched]** Third-party synthesis of KubeCon EU 2026: agentic AI was *the* story, agents proliferating, and "**nobody has quite figured out how to manage and secure them inside Kubernetes yet**". + +**Every AI-SRE incumbent stops at advise/diagnose or autonomy-first — none claims governed, approval-gated action as a neutral primitive:** +- **[fetched]** **kagent** (CNCF Sandbox, donated by Solo.io Apr 2025) is a *framework to run agents in-cluster* (agents as CRDs) — a runtime, **not** governance; a natural **MCP client of Sith** ([CNCF kagent](https://www.cncf.io/blog/2025/04/15/cncf-welcomes-kagent/)). +- **[fetched]** **HolmesGPT** stops at "natural-language diagnosis and remediation steps" — launch post has zero mentions of human/approval/guardrail/RBAC. +- **[fetched]** **Komodor** Klaudia multi-agent (GA 2026-03-18): the press release has **no mention** of approval/human-in-the-loop/guardrail/policy/governance/audit/RBAC — autonomy-first. +- **[fetched]** **Rancher "Liz" crew** (KubeCon EU 2026) is framed as **advisory** (insights/recommendations), no language about executing changes, no approval/guardrail primitives. + +**MCP gateways enforce auth + tool allowlists + audit — not fleet-aware, approval-gated action:** +- **[fetched]** **Kong AI Gateway 3.13** MCP Tool ACLs = identity-based per-tool allow/deny, default-deny, audit — Kong's own scope statement mentions **no human-approval, no HITL, no stop-mid-action** ([Kong MCP ACLs](https://konghq.com/blog/product-releases/mcp-tool-acls-ai-gateway)). +- **[fetched]** **Solo agentgateway + kagent** (closest K8s-native stack) = OIDC + token exchange + AccessPolicies binding which agent calls which tool; **no approval gates or action controls**; governance is in the **commercial** tier, leaving **OSS-native agent governance unoccupied** ([Solo kagent security](https://docs.solo.io/kagent/latest/security/)). +- **[fetched]** **MintMCP** claims generic tool-level approval gating and read/write enforcement — but **"generic per-tool rules with no fleet or cluster semantics"**; its only K8s reference is hosting its own connectors ([MintMCP agent gateway](https://www.mintmcp.com/blog/agent-gateway)). +- **[fetched]** **Permit.io** ships an "Access Request MCP" for human-approval-before-action — but **domain-generic** (copilots, support bots); the page has **no mention of Kubernetes/clusters/fleet** ([Permit.io](https://docs.permit.io/ai-security/access-request-mcp/overview/)). LangSmith/Langfuse give **observability/tracing**, not control. + +**The precise white space (triangulated from all of the above):** *approval-gated, fleet-aware, blast-radius-conscious governance of agent actions on Kubernetes clusters, as a vendor-neutral OSS primitive.* Generic approval gating exists (Permit.io, MintMCP). Tool allowlists + audit exist (Kong, Solo). Agent runtimes exist (kagent). Autonomy-first AI-SRE exists (Komodor, Rancher, Holmes). **The intersection — typed cluster-verbs + multi-approver + canary waves + abstention + signed dispatch + decision-ledger, applied identically to humans and to any MCP-client agent — is claimed by no one.** That is Sith's position, and the KubeCon EU 2026 read ("nobody has figured out how to manage/secure agents in K8s yet") is the market saying the door is open. + +## 5. What this means for Sith's plan + +- **Connector framework:** out-of-process gRPC, SDK-first, three fixed kinds, one canonical connector per tool, minor-additive versioning. This is a *fast-follow* deliverable — the wedge ships with hand-written adapters for the day-1 six; the framework generalizes them once the shape is proven (avoid Backstage's premature-ecosystem trap). +- **MCP server is the wedge's amplifier, not a separate bet:** because Sith enforces server-side through the same PEP, exposing the fleet as a governed MCP server makes *every* external agent (Claude Code, Codex, kagent) inherit the governance for free. That is the "governed MCP gateway to your whole fleet" position, and §4 shows it is empty. +- **Sith as MCP *client*** (calling kagent/Grafana MCP/GitHub MCP) is a later convenience, not the wedge. +- **The charter's AI stance is confirmed by evidence:** annotations are hints (verified), enforcement must be server-side (verified), elicitation is the HITL primitive (verified), and no incumbent occupies governed fleet action (verified). Hold the line: closed verb vocabulary, no shell, AI as client of the same PEP. diff --git a/docs/research/market-and-form-factor.md b/docs/research/market-and-form-factor.md new file mode 100644 index 0000000..126eebc --- /dev/null +++ b/docs/research/market-and-form-factor.md @@ -0,0 +1,268 @@ +# Sith — Market, tool landscape, and form factor (workstreams A · B · C · F · G) + +**Status:** research · **Date:** 2026-07-09 · **Provenance:** salvaged and consolidated from a deep-research fan-out (100+ verified sub-agent results per workstream, run 2026-07-08) plus a small amount of hand gap-filling (2026-07-09). Every load-bearing claim carries a primary-source link; vendor-commissioned figures are labelled as such. This document consolidates three workstream drafts — practitioner pains (A), form factor & developer experience (C), and the tool / cost / multi-cloud landscape (B · F · G). Companions: [identity-connections-security.md](identity-connections-security.md) (run D), [integrations-and-ai-governance.md](integrations-and-ai-governance.md) (run E), and the synthesis [USE-CASE-AND-SHAPE.md](USE-CASE-AND-SHAPE.md). + +**Contents** +- [Part 1 — Practitioner pains: global, China, India](#part-1) +- [Part 2 — Form factor and DX: the "Lens IDE" question](#part-2) +- [Part 3 — Tool landscape, cost, and multi-cloud gap map](#part-3) + +--- + + + +## Part 1 — Practitioner pains: global, China, India + +**Date:** 2026-07-08 · **Method:** web research with fetched primary sources; verbatim quotes pulled from each source. Load-bearing claims were re-checked by hand where flagged. Survey figures are cited with the exact page/figure where the source provides one. Vendor-commissioned research is labelled as such. + +This file answers: which pains are frequent and severe enough that a fleet-operations tool must kill them, and what China and India add. + +--- + +## 1. The quantitative baseline: fleets are real, and they hurt + +| Fact | Figure | Source (fetched) | +|---|---|---| +| Fleet size is mainstream, not exotic | "The average Kubernetes adopter now operates more than 20 clusters"; 56% have >10 | [Spectro Cloud, State of Production Kubernetes 2024](https://www.spectrocloud.com/blog/ten-essential-insights-into-the-state-of-kubernetes-in-the-enterprise-in-2024) (n=416, vendor survey) | +| Same figure from a second, independent vendor | "A typical enterprise now runs more than 20 clusters, with nearly half operating across more than four environments"; 37% manage >100 clusters, 12% >1,000 | [Komodor 2025 Enterprise Kubernetes Report](https://komodor.com/blog/komodor-2025-enterprise-kubernetes-report-finds-nearly-80-of-production-outages/) (customer telemetry, not a survey) | +| Fleets span environments | Half of businesses run clusters in 4+ environments (clouds, DCs, edge) | Spectro Cloud 2024, above | +| Hybrid is the norm | 86% deploy across both public and private cloud | [Portworx, Voice of Kubernetes Experts 2024](https://portworx.com/wp-content/uploads/2024/06/The-Voice-of-Kubernetes-Experts-Report-2024.pdf) (n=527, 500+ employee orgs, Dimensional Research) | +| Estates skew self-managed and multi-cloud | 59% self-managed on-prem and 59% self-managed public cloud; 37% use 2 cloud providers, 26% use 3 | [CNCF Annual Survey 2024 PDF](https://www.cncf.io/wp-content/uploads/2025/04/cncf_annual_survey24_031225a.pdf) (n=689–750 depending on question) | +| Complexity is the headline pain | "Three quarters of businesses that use Kubernetes today say their adoption of K8s has actually been inhibited by the complexity" | Spectro Cloud 2024, above | +| Change is the outage engine | "79% of production issues originate from a recent system change" | Komodor 2025, above | +| Toil is quantified | >60% of ops time spent troubleshooting; 64+ workdays/year lost; median MTTD ~40 min, MTTR >50 min for high-impact outages; 38% see high-impact outages weekly | Komodor 2025, above (telemetry sample caveat) | +| Cost pressure is rising | Nearly two-thirds report K8s TCO grew and face more cost pressure than a year ago, with poor visibility into future cost | Spectro Cloud 2024, above | +| Waste is endemic | ">82% of Kubernetes workloads are overprovisioned (65% use less than half of the CPU and memory they request)" | Komodor 2025, above | +| Lock-in anxiety is measurable | 55% "already feel locked in"; more than half worry about vendors shutting down | Spectro Cloud 2024, above | +| Demand for a central plane exists | 71% say a unified/centralized platform would greatly benefit them; 43% specifically want streamlined hybrid/multi-cloud management across Kubernetes environments | Portworx 2024, above (framed around data services — Portworx's domain; treat as directional) | +| The buyer function exists almost everywhere | 96% of surveyed enterprises have a platform-engineering function; drivers include cost (49%) and security mandates (43%) | Portworx 2024, above | + +Two honest caveats on this table. First, three of the five sources are vendor-commissioned; the figures converge (20+ clusters, 4+ environments, complexity/cost as top pains), which is why they are usable, but no single number should be treated as precise. Second, the CNCF survey's own challenge ranking (Figure 8: cultural change 46%, CI/CD 40%, training 38%, security 37%, monitoring 36%, complexity 35%) shows people/skills pains now rank alongside technical ones — a tool that requires deep OCM or fleet-theory expertise to install will fail the very population that needs it. + +### What the CNCF survey says about *adopting a new OSS tool* (directly actionable for Sith) + +The top 2024 barriers to running open-source projects in production: fear the project becomes inactive (46%, +9 pts YoY), too complex to understand or run (46%, +13 pts), lack of documentation (45%, +5 pts) ([CNCF 2024 PDF](https://www.cncf.io/wp-content/uploads/2025/04/cncf_annual_survey24_031225a.pdf)). Security-vulnerability concern *fell* to 29%. The adoption killers for a new tool are perceived abandonment risk, complexity, and thin docs — not security scanners. Sith's counter must be structural: trivial install, excellent docs, visible release cadence. + +## 2. Ranked pains a fleet tool must kill + +Ranked by convergence of survey figures and community evidence. (The community-thread search lane of this research was partially rate-limited; where thread-level citations are thin the ranking leans on the survey base above and the tool-exodus evidence in [Part 2](#part-2).) + +1. **"What is happening across my clusters right now?" — fleet-wide visibility and correlation.** 20+ clusters across 4+ environments with per-cluster consoles means N logins to answer one question. The Portworx 71%/43% centralization demand and the Komodor MTTD figures quantify it. Single-cluster tools structurally cannot answer "which clusters run image X" or "where is `payments` degraded". +2. **"What changed, where?" — change attribution across the fleet.** 79% of production issues stem from a recent change (Komodor). A fleet tool that can answer "what changed in the last hour, across which clusters, by whom" attacks the single largest outage cause. +3. **Doing the same action on N clusters safely.** The action side of pain 1. Today it is a shell loop over kubeconfigs (no gates, no audit, no rollback) or a heavyweight platform (Rancher/ACM). Severity evidence is indirect but strong: change-driven outages (79%), weekly high-impact outages (38%), and the absence of any OSS primitive for gated fan-out (see [Part 3](#part-3) §OCM). +4. **Credential and access sprawl.** Kubeconfig-per-cluster with exec plugins, shared admin credentials, no per-user attribution. Survey proxy: security named by 37% (CNCF). The predecessor's own failure (central god-kubeconfig) and the entire brokered-access market (Teleport et al., see [identity-connections-security.md](identity-connections-security.md)) exist because of this pain. +5. **Cost visibility across clusters and clouds — including GPU.** Two-thirds face rising TCO with poor visibility (Spectro Cloud); >82% overprovisioned (Komodor); GPU is the extreme case — see §3. +6. **Complexity/skills mismatch.** 75% say complexity inhibited adoption (Spectro Cloud); training gaps at 38% (CNCF). A fleet tool must *reduce* the expertise required, not add a new discipline. +7. **Vendor lock-in anxiety.** 55% feel locked in (Spectro Cloud); the 2025 SUSE/Rancher repricing (competitor-reported 4–9x increases — [Portainer's account](https://www.portainer.io/blog/suse-rancher-price-hike-why-enterprises-are-searching-for-alternatives-in-2025), cross-check against SUSE's [shop page](https://www.suse.com/shop/suse-rancher-prime/)) and the Kubecost/IBM and Lens/Mirantis consolidations feed it. Vendor-neutral OSS is a distribution advantage here, not a nicety. + +## 3. MLOps / GPU fleet pains + +- **GPU waste is extreme and measured.** CAST AI's 2026 analysis of tens of thousands of production clusters: **average GPU utilization 5%** ("95% of GPU capacity is doing nothing"), vs 8% CPU and 20% memory ([CAST AI 2026 State of Kubernetes Optimization](https://cast.ai/press-release/2026-state-of-kubernetes-optimization-report/); vendor telemetry). The ClearML/AIIA survey (~1,000 AI leaders): only 7% achieve >85% peak GPU utilization; 74% dissatisfied or only moderately satisfied with job-scheduling tools; 93% say easy self-serve compute would substantially raise productivity ([ClearML State of AI Infrastructure at Scale 2024](https://clear.ml/blog/the-state-of-ai-infrastructure-at-scale-2024)). +- **Per-team GPU cost attribution is formalized as a discipline but only single-cluster.** The FinOps Foundation working group codifies namespace-as-cost-centre labeling enforced at admission, showback-then-chargeback, and MIG for right-sizing ("sub-5 GB models hogging an entire A100") — and its paper **covers only single-cluster scenarios; cross-cluster allocation is unaddressed** ([FinOps WG: Scaling Kubernetes for AI/ML with FinOps](https://www.finops.org/wg/scaling-kubernetes-for-ai-ml-workloads-with-finops/)). That gap is exactly the fleet layer. +- **In-cluster GPU scheduling is being commoditized; the fleet layer is not.** NVIDIA open-sourced its Run:ai-derived KAI-Scheduler (CNCF Sandbox) for fairness across teams at thousand-node scale ([KAI-Scheduler](https://github.com/kai-scheduler/KAI-Scheduler)). The defensible pains for a fleet tool sit *above* the scheduler: federated per-team usage/cost across N clusters, and governed actions on training/inference fleets. +- **ML platform teams re-invent governed access.** ZenML's platform guidance: data scientists should never hold raw cluster credentials; access should be centrally brokered ([ZenML on 8xH100 multi-tenancy](https://www.zenml.io/blog/managing-mlops-at-scale-on-kubernetes-when-your-8xh100-server-needs-to-serve-everyone)). MLOps is thus a *user segment* of the same governed-access/action wedge — not a separate product. +- 54% of enterprise K8s orgs already run AI/ML on Kubernetes (Portworx 2024, above), so GPU columns in the fleet model serve a mainstream slice, not a niche. + +## 4. China + +The CNCF survey cannot support China conclusions (~3% of respondents China-HQ'd — the survey says so itself, [2024 PDF](https://www.cncf.io/wp-content/uploads/2025/04/cncf_annual_survey24_031225a.pdf) demographics). The evidence below is from Chinese-language primary sources (vendor docs, practitioner tutorials) and is labelled accordingly. Direct practitioner-sentiment evidence at scale is thin in public sources; that gap is flagged rather than papered over. + +**4.1 Air-gap/offline is a first-class, routine deployment mode — not an edge case.** +- KubeSphere documents air-gapped installation as a standard path (KubeKey artifact + private Harbor) ([KubeSphere docs, 离线安装](https://kubesphere.io/zh/docs/v3.3/installing-on-linux/introduction/air-gapped-installation/)). +- A representative practitioner walkthrough (运维有术 series on Tencent Cloud's developer community) builds the offline bundle on a connected node and carries it in; the full artifact is ~13 GB and populates 124 image repos in self-hosted Harbor; the topology is described as a 1:1 replica of a small production environment ([cloud.tencent.com/developer/article/2419243](https://cloud.tencent.com/developer/article/2419243)). +- The assumed baseline is that public registries are unreachable: the tutorial *verifies* isolation by showing `docker.io` image pulls failing on Chinese public DNS. A separate xinchuang field report states plainly that hub.docker.com "is currently not accessible from within China — figure out your own way" ([CSDN, 信创适配实战](https://blog.csdn.net/yztezhl/article/details/139698545), June 2024). + +**4.2 Compliance shapes tooling: MLPS 2.0 (等保 2.0) is operationalized, not theoretical.** +- Alibaba ACK ships MLPS 2.0 Level 3 hardening as a cluster-creation option implementing GB/T 22239-2019 ([ACK 等保加固说明](https://help.aliyun.com/zh/ack/ack-managed-and-ack-dedicated/security-and-compliance/ack-reinforcement-based-on-classified-protection)). Level-3 hardening mandates: **role separation** (ACK creates distinct `ack_admin`, `ack_audit`, `ack_security` users), **tamper-resistant, backed-up audit records** ("应对审计记录进行保护,定期备份,避免受到未预期的删除、修改或覆盖"), and **no root SSH** — i.e., hardened Chinese estates structurally favor governed, audited, agent-mediated access over interactive credentials. +- The MLPS model explicitly allows substituting self-attested equivalent controls ("如果有其他方式,可自行举证并忽略此项") — a self-hosted OSS tool can legitimately slot into an MLPS-graded estate. +- **Xinchuang (信创) localization** adds: domestic OSes (openEuler/OpenAnolis, or Kylin/UOS when customers demand), domestic ARM CPUs (Kunpeng/Phytium), hence **linux/arm64 multi-arch images are a hard requirement**, and tolerance for older pinned K8s versions (the field report ran v1.24 in mid-2024) ([CSDN field report](https://blog.csdn.net/yztezhl/article/details/139698545)). + +**4.3 The cloud substrate is conformant Kubernetes.** Alibaba ACK, Huawei CCE and Tencent TKE all hold current CNCF Certified Kubernetes conformance (hand-verified in [cncf/k8s-conformance](https://github.com/cncf/k8s-conformance): `v1.32/alicloud` = "Alibaba Cloud Container Service for Kubernetes v1.32.0"; `huawei-cce` submissions continuously v1.29–v1.34; `v1.34/tencentcloud` = "Tencent Kubernetes Engine v1.34.1"). All three issue standard kubeconfigs (RAM/IAM/CAM-integrated) with private-VPC endpoints ([ACK kubeconfig](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/obtain-the-kubeconfig-file-of-a-cluster-and-use-kubectl-to-connect-to-the-cluster), [CCE permissions](https://support.huaweicloud.com/intl/en-us/usermanual-cce/cce_10_0187.html), [TKE connecting](https://www.tencentcloud.com/document/product/457/30639)). Rancher already imports ACK/CCE/TKE via its outbound agent ([SUSE announcement](https://www.suse.com/c/rancher_blog/announcing-added-support-for-leading-kubernetes-services-in-china/)) — feasibility proof for the same pattern. +- Chinese-language practitioner comparisons of multi-cluster tools (Rancher vs KubeSphere vs Karmada) turn on cluster import, network reachability and unified permissions ([Zhihu 多集群管理工具对比](https://zhuanlan.zhihu.com/p/539203985), [Kubernetes多集群管理之路](https://zhuanlan.zhihu.com/p/584378217) — the latter names heterogeneous cluster types, 10k-node scale spread, per-region compliance differences, version drift, and "scattered permissions with uncontrolled security risk" as the operating reality). + +**4.4 What this means for Sith in China.** A US-hosted SaaS control plane (Komodor, Datadog, Lens-with-account) is structurally disadvantaged: GFW/egress policy, data-residency posture, and procurement all push toward self-hosted. Sith fits **if and only if** it ships: (1) fully offline installation (single bundle, Zarf-style, no phone-home — see [Part 3](#part-3) on Zarf); (2) linux/arm64 + x86 multi-arch images and support for openEuler/Kylin-class hosts; (3) registry-relocatable images (no hardcoded docker.io/gcr.io pulls); (4) tamper-evident audit logs and admin/auditor role separation (maps directly to MLPS L3); (5) hub-and-spoke that works entirely inside one network boundary (the hub is self-hosted in-country; outbound-only spokes work within/behind the boundary). These are the same properties the federated design already targets — China raises their priority from "nice" to "mandatory for the market". + +## 5. India + +- **The buyer base is large and institutional.** 1,700+ GCCs (2,975+ units) generating $64.6B and employing ~1.9M as of FY2024, projected to 2,100–2,200 centers and $99–105B by 2030 ([Nasscom–Zinnov India GCC Landscape 2026 PDF](https://media.zinnov.com/wp-content/uploads/2026/05/zinnov-nasscom-india-gcc-landscape-2026-report.pdf)). GCCs increasingly own engineering/platform functions for global parents — i.e., they operate exactly the multi-cluster, multi-environment estates Sith targets. IT-services majors position Kubernetes as the multi-cloud abstraction across client estates ([Wipro on K8s-native multi-cloud](https://www.wipro.com/blogs/sreekanth-nyamars/kubernetes-native-design-thinking-realizing-true-multi-cloud-adoption/) — vendor voice; treat as estate-shape evidence, not pain ranking). The service-integrator profile adds a specific requirement: **hard isolation between client estates in one operator's tooling** — Sith's workspace model must treat "many clients, one operator" as a first-class shape. +- **Regulatory pull is procurement-level, not absolute.** DPDP Act 2023 (+ DPDP Rules 2025) uses a blacklist approach to cross-border transfer rather than blanket localization, but the government can pin specified data of "significant data fiduciaries" in-country; phased compliance runs to ~May 2027 with penalties to ₹250 crore ([EY analysis](https://www.ey.com/en_in/insights/cybersecurity/decoding-the-digital-personal-data-protection-act-2023)). The hard residency floor is sectoral: RBI's 2018 directive requires payment-system data stored only in India ([Google Cloud RBI compliance page](https://cloud.google.com/security/compliance/rbi-india)). BFSI is a top GCC vertical, so in-country self-hosting is often a de-facto requirement; SaaS buyers increasingly ask for India hosting or on-prem options ([Wattlecorp DPDP-for-SaaS guide](https://www.wattlecorp.com/saas-providers-guide-to-dpdp-act-india/) — mid-tier source). Self-hostable OSS derisks all of this by construction. +- **Cost sensitivity + skills profile.** GCC hiring data indicates a 40%+ skills gap in tech roles and a cited 55–60% deficit in cloud-native expertise, with salary premiums for K8s/CI-CD engineers ([Savannah HR aggregation of NASSCOM/Deloitte figures](https://savannahr.com/blog/top-8-gcc-skills-india-2026/) — recruiting-blog source, verify against the underlying reports before quoting numbers). Implication: free OSS core with radically simple install and opinionated safe defaults beats "powerful but expert-only". Direct India practitioner sentiment on fleet tooling is thin in public sources — flagged as a gap; the estate-shape and regulatory evidence above is the reliable part. + +## 6. Conclusions carried into the synthesis + +1. The pains Sith must kill, in order: fleet-wide visibility/correlation; change attribution; safe fan-out actions; access/credential sprawl; fleet cost overlay (GPU included). These map 1:1 onto read federation, action federation, and a cost read-integration — the charter's wedge survives contact with the demand evidence. +2. OSS-adoption barriers (abandonment fear, complexity, docs) mean the *form* of the product decides adoption as much as the wedge. A 10-minute, single-binary first-run is not polish; it is the counter to the #1 and #2 adoption barriers. +3. China: air-gap bundle, multi-arch, registry relocation, tamper-evident audit, role separation — mandatory, and cheap if designed in early. +4. India: multi-client workspace isolation and self-hostability are the fit; price the OSS core at zero and keep the install trivial. +5. MLOps is a segment, not a separate product: GPU util/cost columns in the fleet model + the same governed actions. + + +--- + + + +## Part 2 — Form factor and DX: the "Lens IDE" question + +**Date:** 2026-07-08 · **Method:** fetched primary sources with verbatim quotes; GitHub figures read from the live pages/API on 2026-07-08; the Kubernetes-Dashboard-to-Headlamp claim, k9s stats, and China-cloud conformance were re-verified by hand. + +This file answers: what happened to Lens and where users went; what a local Kubernetes tool must have; what makes tools feel effortless; what technology to build on; and whether local + federated should be one product or two. + +--- + +## 1. The Lens story, verified in both directions + +The "Lens went paywall/telemetry" narrative is **real but more precise than the folklore**: + +| Date | Event | Source | +|---|---|---| +| 2020-08-13 | Mirantis acquires Lens ("world's most popular Kubernetes IDE", MIT-licensed, ~35k users); founder quoted promising it "would remain vendor neutral and open source" | [Mirantis press release](https://www.mirantis.com/company/press-center/company-news/mirantis-acquires-lens-the-worlds-most-popular-kubernetes-ide/) | +| ~2022-05/06 (Lens 5.5.x) | Mandatory, non-skippable **Lens ID login** lands; team concedes it "could have done better at communicating the change" | [lensapp/lens#5444](https://github.com/lensapp/lens/issues/5444) | +| 2022-07 | **Lens 6 subscription model**: Lens Pro $19.90/user/mo; free Personal tier restricted to individuals and orgs under $10M revenue/funding | [Mirantis announcement](https://www.mirantis.com/blog/lens-pro-vision-for-the-future-new-subscription-model-new-features-available/); backlash: [HN "Lens goes subscription only"](https://news.ycombinator.com/item?id=32269258), [HN on the acquisition/login](https://news.ycombinator.com/item?id=32408122) | +| 2022-12/2023-01 (6.3.0) | Pod **logs/shell menus removed from the open-source build** (moved to an extension); the single most-cited exodus trigger | [lensapp/lens#6823](https://github.com/lensapp/lens/issues/6823); [OpenLens README](https://github.com/MuhammedKalkan/OpenLens) ("type `@alebcay/openlens-node-pod-menu` into the Extensions page") | +| 2024-03 | **Lens closes its source**; OpenLens build repo freezes (last release v6.5.2, 2023-06-30; repo README: "Lens Closed its source code. So please do not expect any more updates.") | [OpenLens repo](https://github.com/MuhammedKalkan/OpenLens); [HN thread](https://news.ycombinator.com/item?id=39811772); alternatives catalogued in [lensapp/lens#8008](https://github.com/lensapp/lens/issues/8008) | +| Today (verified 2026-07-08) | Lens pricing: Personal **free** (under $10M revenue/funding) and still includes multi-cluster management, metrics, logs, terminal, Helm, port-forwarding, resource editing; Plus $25/user/mo (AI copilot, EKS/AKS auto-discovery, Security Center); Enterprise custom. Telemetry: Lens "may automatically communicate with Mirantis servers" for updates/usage tracking, **opt-out available**, vendor asserts no kubeconfigs/secrets uploaded | [lenshq.io/pricing](https://lenshq.io/pricing) (k8slens.dev/pricing redirects here); [Lens licensing/telemetry FAQ](https://docs.k8slens.dev/faq/subscription-and-licensing/) | + +So the adversarial check lands here: **core features are mostly still free; what users actually revolted against was the account wall, the trust break (closed source after a vendor-neutrality promise), telemetry-by-default, and the removal of logs/shell from the OSS build.** Those four things — not any missing feature — created the exodus. A new tool wins that audience by structural promises: no account, no telemetry, open source, logs/exec in core forever. + +### Where the exodus went (traction figures as of 2026-07-08) + +| Tool | What it is | Traction | Gap it leaves | +|---|---|---|---| +| [Freelens](https://github.com/freelensapp/freelens) | MIT fork of OpenLens (Electron, TypeScript), no account/telemetry | **5.3k stars in ~2 years** (repo created 2024-06-19), v1.10.3 released 2026-07-07, ~monthly cadence, 29 releases | Inherits Lens's one-cluster-at-a-time UX and Electron weight; a continuation, not a rethink | +| [OpenLens builds](https://github.com/MuhammedKalkan/OpenLens) | Login-free Lens binary | 4.4k stars, **dead since 2023-06** | Demand signal for "no login" — 4.4k stars for a *build repo* | +| [k9s](https://github.com/derailed/k9s) | Terminal UI, single Go binary | **34.1k stars**, v0.51.0 (2026-06-06), Apache-2.0, `brew install derailed/k9s/k9s` | **One context at a time** (`:ctx` to switch); no aggregated fleet view; TUI ceiling for sharing/visualization | +| [Headlamp](https://github.com/kubernetes-sigs/headlamp) | Kubernetes SIG-UI web UI + desktop app, plugin system | 6.8k stars; v0.43.0 (2026-06-16); monthly releases; AI assistant via MCP; Artifact Hub plugin catalog ([2025 highlights, kubernetes.io](https://kubernetes.io/blog/2026/01/22/headlamp-in-2025-project-highlights/)) | Per-cluster-centric UX (multi-cluster registration exists; ClusterProfile inventory is alpha); no fleet correlation; no governed actions | +| [Aptakube](https://aptakube.com/) | Closed-source Tauri desktop client | Vendor claims "thousands" of users; **$9/mo personal / $7/seat teams, 15-day trial, no free tier**; installers 15–28 MB | Proves the paying gap: its headline claim is being "the **only** Kubernetes UI that can connect to multiple clusters simultaneously" and aggregate resources in one view, plus "no extra configuration… data never leaves your machine" ([aptakube.com](https://aptakube.com/), [lens-alternative page](https://aptakube.com/lens-alternative)). Closed and paid — the OSS slot for exactly this is **empty** | + +**The pivotal ecosystem event (hand-verified):** the official **Kubernetes Dashboard is archived**, and the kubernetes.io blog names Headlamp the way forward *explicitly because of* "multi-cluster visibility … and flexible deployment options that work both in-cluster and on the desktop" ([From Kubernetes Dashboard to Headlamp, 2026-06-01](https://kubernetes.io/blog/2026/06/01/dashboard-to-headlamp/), authored by Will Case/Headlamp). Read as market evidence: the single-mode, single-cluster web console lost; the dual-mode, multi-cluster tool became the community default. Any Sith local mode is therefore **not** competing with a vacuum — Headlamp is CNCF-blessed and improving monthly. Sith's local mode must not be "another general console"; it must be the **fleet** view Headlamp doesn't center on (aggregation, correlation, staleness, governed actions) — see §5. + +### Table-stakes for a Lens-class local tool (ranked by how often the evidence cites them) + +1. Multi-cluster from existing kubeconfig, zero config ("if you're already using kubectl, it just works" — the Aptakube pitch). +2. Pod logs + exec/shell in core (their removal *created* the OpenLens exodus — [#6823](https://github.com/lensapp/lens/issues/6823)). +3. No account/login wall (4.4k stars on a build repo whose only feature was deleting the login). +4. No telemetry, or opt-in only; local-only data ("never leave your machine"). +5. Fast on big clusters; low memory (Electron complaints are constant in Lens-alternative threads). +6. Resource browse + YAML edit, port-forward. +7. Open source under a permissive license, active cadence (CNCF-survey adoption barriers: abandonment fear, docs). +8. Aggregated multi-cluster single view — the one thing users can otherwise only buy (Aptakube). + +## 2. Why the effortless tools feel effortless (transferable mechanics) + +- **Latency budget:** 0.1 s = direct manipulation; 1 s = flow intact; 10 s = attention lost ([Nielsen/NN-g response-time limits](https://www.nngroup.com/articles/response-times-3-important-limits/)). Palette open, fuzzy search, and view switches must land under ~100 ms — only achievable when rendering from a **local cache/store**, not a per-keystroke round-trip to N API servers. +- **Local-first with background sync (the Linear mechanic):** the server is a sync target, not the UI's source of truth; the client hydrates a local store, every query hits it first, pages render in <50 ms with no spinners; deltas reconcile asynchronously ([How is Linear so fast](https://performance.dev/how-is-linear-so-fast-a-technical-breakdown); practitioner corroboration: [local-first rabbit hole](https://bytemash.net/posts/i-went-down-the-linear-rabbit-hole/)). Kubernetes has the perfect substrate for this: **watch streams into a local informer cache**. This is the single most important DX decision for Sith's local mode: cache-first render + staleness stamps, never spinner-first. +- **Keyboard-first + command palette:** the cmd-K bar bridges GUI discoverability and CLI speed and is the canonical pattern for serving both kubectl power users and GUI users in one product ([Maggie Appleton, Command K Bars](https://maggieappleton.com/command-bar)). k9s is the in-domain proof that keyboard-first wins operators (34.1k stars). +- **Install friction:** k9s (`brew install`, one binary, no server, no account) and Freelens (brew cask + winget/scoop/flatpak/deb/rpm, arm64+amd64) define the funnel. Tailscale's "install, sign in, connected — value on first run" is the canonical 10-minute-wow articulation ([why-tailscale](https://tailscale.com/why-tailscale)). Contrast: "helm install a platform, configure SSO, then see value" is the devops-portal/Backstage adoption cliff. **Sith's local mode must be a single artifact that shows a populated fleet view within minutes of `brew install`.** + +## 3. Desktop technology: Electron vs Tauri vs neither + +Benchmarks and production reports (all fetched): +- Same-app comparisons: Tauri installer ~2.5 MB vs Electron ~85 MB; idle RAM ~80 MB vs ~120 MB; cold start ~2 s vs ~4 s ([Authme dev, levminer.com](https://www.levminer.com/blog/tauri-vs-electron)). Minimal-app: 8.6 MiB vs 244 MiB bundle; ~172 MB vs ~409 MB with 6 windows; **startup difference negligible** in that test ([Hopp benchmark, 2025](https://www.gethopp.app/blog/tauri-vs-electron)). +- Counterweights: Tauri renders via OS webviews (WebView2/WKWebView/WebKitGTK) → **cross-platform rendering inconsistency** to manage, vs Electron's identical-everywhere Chromium; DoltHub stayed on Electron for packaging gaps (no .appx/.msix, no macOS universal binaries) while still concluding Tauri "eliminates much of the classic Electron bloat" ([DoltHub, 2025-11](https://www.dolthub.com/blog/2025-11-13-electron-vs-tauri/)); post-migration retrospective: [Fluxzy five months after](https://www.fluxzy.io/resources/blogs/electron-to-tauri-migration-fluxzy-desktop). +- Category proof: **Aptakube ships on Tauri** in exactly this product class, marketing the small/fast footprint against Electron-based Lens; other K8s clients on Tauri exist (JET Pilot, Kunobi — [awesome-tauri](https://github.com/tauri-apps/awesome-tauri)); Tauri's sidecar mechanism cleanly wraps an existing backend binary (DoltHub, Hopp). + +**Recommendation:** don't make the webview choice the first decision. The proven architecture in this exact category is **Headlamp's**: one Go backend + one web frontend, identical in both modes, with the desktop app being a thin shell over the same code ([Headlamp architecture docs](https://headlamp.dev/docs/latest/development/architecture/)). Sith should ship a **single Go binary** whose `sith ui` serves the local web UI from the embedded frontend (k9s-grade install friction, no app-store/codesigning tax on day one), plus a first-class CLI. A **Tauri** shell (not Electron — memory/footprint evidence above, and Lens-refugee sensitivity to Electron bloat) is a fast-follow for dock presence/deep-OS integration, wrapping the same binary as a sidecar. This sequences the risk: the web UI must exist in both form factors anyway; the wrapper is additive. + +## 4. One product or two? The dual-mode precedents + +| Precedent | Shape | Verdict | +|---|---|---| +| **Headlamp** | Same Go backend + React frontend runs as single-user desktop app (local kubeconfigs) and as in-cluster multi-user web deployment; docs state the modes are "not mutually exclusive" — individuals use desktop while the org runs in-cluster ([installation docs](https://headlamp.dev/docs/latest/installation/), [architecture](https://headlamp.dev/docs/latest/development/architecture/)) | **Strongest validation.** The dual mode is *why* it won the Dashboard succession ([kubernetes.io, 2026-06-01](https://kubernetes.io/blog/2026/06/01/dashboard-to-headlamp/)) | +| **Portainer** | One server binary; solo homelab user runs it alone; the same server accepts agents for centralized multi-environment management; Business layers RBAC on the same core ([architecture docs](https://docs.portainer.io/start/architecture)) | Validates "same product, agents arrive later, governance is a layer" | +| **Grafana** | Same OSS core run locally/self-hosted; Cloud adds hosting + governance features ([oss-vs-cloud](https://grafana.com/oss-vs-cloud/)) | Validates monetizing/governing a tier above one core | +| **Teleport** | Desktop client (Teleport Connect, Electron) lives in the same monorepo as the server/Web UI, sharing infrastructure ([Teleport Connect docs](https://goteleport.com/docs/connect-your-client/teleport-clients/teleport-connect/)) | Weaker (client-of-server, not standalone local mode), but same-repo/shared-UI economics hold | + +**Conclusion: one product, one binary, two modes.** Every relevant precedent shares code between local and served modes; the community's own Dashboard→Headlamp migration explicitly rewarded the dual mode. Two separate products would double surface area (the devops-portal failure mode) and break the land-and-expand path (§5). + +## 5. The recommended shape for Sith + +- **One Go binary, three run modes:** `sith` (CLI verbs), `sith ui` (local web UI on localhost, kubeconfig-direct, single user, zero config, no account, no telemetry), `sith hub` (the federated control plane serving the *same* UI multi-user with workspaces/governance). One frontend; the UI renders identically from a local cache (direct mode) or the hub's fleet model (federated mode). +- **The local mode's center of gravity is the fleet, not the pod.** Aggregated all-clusters resource views, fleet search/correlation ("which clusters run image X", "where is payments degraded"), staleness stamps, and the same typed-verb actions with dry-run/diff — locally self-approved, but the same intent model that later gains real governance. Per-pod table stakes (logs, exec, port-forward, YAML edit) must exist because their absence created the Lens exodus — but they are commodity K8s API calls, not integrations, and Sith should not chase Headlamp/k9s feature-for-feature beyond them. +- **The wow:** `brew install sith && sith ui` → all kubeconfig contexts detected → one aggregated fleet view with cmd-K fuzzy search across every cluster, under 10 minutes, offline-capable, nothing leaves the machine. +- **The expand:** when the team needs shared visibility and real approvals, the same binary becomes the hub; clusters upgrade from "direct (my kubeconfig)" to "minion (outbound OCM agent)" without the user relearning anything. Local mode is the top of the funnel for the governed wedge — not a second product. + + +--- + + + +## Part 3 — Tool landscape, cost, and multi-cloud gap map + +**Date:** 2026-07-08 · **Method:** fetched primary sources with quotes; facts marked *(repo-verified)* were web-verified in this repository's [`COMPETITIVE.md`](../../COMPETITIVE.md) during the July 2026 planning pass and re-used here; hand-verified items are marked. Where pricing is quote-based or was not directly fetched, the row says so rather than inventing numbers. + +The question this file answers: what does each incumbent actually provide and charge, and what is missing across *all* of them — the white space Sith can own. + +--- + +## 1. Local / single-operator clients + +Covered in depth in [Part 2](#part-2). Summary of the gap: **k9s** (34.1k stars, one context at a time — hand-verified), **Lens** (account wall, telemetry-with-opt-out, Plus $25/user/mo; Personal free under $10M), **OpenLens** (dead 2023), **Freelens** (healthy MIT fork, Electron, Lens-lineage single-cluster UX), **Headlamp** (SIG-UI, dual-mode, per-cluster-centric, plugin system), **Aptakube** (closed, $9/mo, the only aggregated multi-cluster single view), **kubectl** (the substrate everyone shells out to; no fleet semantics beyond contexts). **White space: an OSS, no-account, no-telemetry, aggregated multi-cluster client with fleet-level search/correlation.** Nobody occupies it; the closest occupant is closed and paid (Aptakube), and the community default (Headlamp) centers on per-cluster views. + +## 2. GitOps / deploy layer (things Sith integrates with, never replaces) + +| Tool | What it is | Relevant boundary facts | +|---|---|---| +| **Argo CD** | CNCF-graduated GitOps CD; REST/gRPC API + `Application` CRDs; its own multi-cluster story is being agent-ified by [argocd-agent](https://github.com/argoproj-labs/argocd-agent) (v0.9.0, 2026-06-04 *(repo-verified)*) | Argo federates *its own* surface only; no cross-tool governed actions. Sith's `argocd.sync|rollback` verbs ride its API | +| **Flux** | CNCF-graduated GitOps controllers; **CRD-only surface** (no API server) — integration = read/patch CRs | Same boundary: reconciler, not an ops control plane | +| **Helm / Kustomize** | Package/overlay standards; Helm state lives in in-cluster release Secrets | Read adapters for inventory ("what release/version is where"), not action targets in v1 | +| **Kargo** ([akuity/kargo](https://github.com/akuity/kargo)) | GitOps *promotion* orchestration; Apache-2.0, v1.10.8 (2026-06-25), ~3.4k stars; open-core (Kargo Enterprise by Akuity, inquiry pricing) | **Closest OSS analog to "governed promotion"** — stages with gates and Git as audit trail — but it promotes *artifacts through environments* via GitOps; it is Argo-ecosystem-coupled and does not execute typed live operations (restart/scale/drain) across heterogeneous fleets. The repo page itself doesn't substantiate multi-approver gates (verify against docs.kargo.io before citing specifics). Boundary, and validation that "promotion with gates" resonates | + +The lesson the predecessor already paid for: re-skinning these tools is negative value. Sith reads their state into the fleet model and dispatches a **closed set of typed verbs** to their APIs — nothing else. + +## 3. Fleet / platform incumbents + +| Tool | Provides | Pricing/positioning | What it does NOT do | +|---|---|---|---| +| **Rancher / Rancher Prime (SUSE)** | Cluster provisioning + import (outbound `cattle-cluster-agent`), RBAC, catalog; Fleet = GitOps at scale; 2026: "Liz" agentic AI expanding to a crew (Linux/Observability/Security/Provisioning/Fleet) + MCP server integration ([SUSE KubeCon EU 2026 announcement](https://www.suse.com/c/kubecon-eu-2026-first-agentic-ecosystem-platform/)) | Prime = paid subscription ([SUSE shop](https://www.suse.com/shop/suse-rancher-prime/), per-node/per-vCPU tiers); 2025 repricing to a vCPU metric with **competitor-reported 4–9x increases driving alternatives-shopping** ([Portainer's account](https://www.portainer.io/blog/suse-rancher-price-hike-why-enterprises-are-searching-for-alternatives-in-2025) — rival-authored, treat accordingly) | No typed-intent action vocabulary, no multi-approver fan-out gates, no abstention semantics; agentic AI is subscription-gated assistance, not a neutral governance primitive. Imports ACK/CCE/TKE ([SUSE](https://www.suse.com/c/rancher_blog/announcing-added-support-for-leading-kubernetes-services-in-china/)) — feasibility proof for agent-based China coverage | +| **OpenShift + ACM (Red Hat)** | The enterprise platform; ACM is the multi-cluster manager **built on OCM upstream** *(repo-verified: OCM underpins ACM)* | Subscription platform; OpenShift-first | ACM governs via Policy CRDs + ManifestWork fan-out; OpenShift-centric, heavyweight, not a neutral primitive for arbitrary conformant clusters; no closed-verb typed actions with per-wave human gates | +| **Open Cluster Management (OCM)** | CNCF Sandbox; hub/spoke registration, Placement, ManifestWork, addons; `cluster-proxy` + `managed-serviceaccount` (both v0.10.0, 2026-02-02) give outbound-only reach + scoped identity *(repo-verified; M0 reproduced hands-on in [`docs/experiments/M0-ocm-falsification.md`](../experiments/M0-ocm-falsification.md))*; OCM v1.3.1 (2026-05-19) | OSS substrate, Red Hat-sponsored | **The critical finding (fetched, [ManifestWorkReplicaSet docs](https://open-cluster-management.io/docs/concepts/work-distribution/manifestworkreplicaset/)):** OCM already ships alpha **Progressive / ProgressivePerGroup rollout strategies** (minSuccessTime, progressDeadline, maxFailures) — so raw canary *sequencing* is NOT white space. What OCM has **no concept of**: approval gates, multi-approver workflows, typed/closed action vocabulary (ManifestWork = arbitrary YAML), operation-level audit ledger, abstention. Governance above OCM is the actual gap, and OCM's addon framework exists precisely so others build it ([CNCF comparison post](https://www.cncf.io/blog/2022/09/26/karmada-and-open-cluster-management-two-new-approaches-to-the-multicluster-fleet-management-challenge/)) | +| **Karmada** | CNCF; multi-cluster *scheduling/propagation*; push and pull modes (so "outbound agent" alone is weak differentiation) | OSS, Huawei-sponsored | Placement engine, not governed ops; same absence of approvals/typed verbs/audit | +| **Clusterpedia** | CNCF Sandbox; multi-cluster resource **search/inventory** (v0.9.1 *(repo-verified)*) | OSS | Read-only; no actions at all; validates fleet-search demand | +| **KubeSphere** | China-origin OSS platform (4.x "LuBan" architecture), first-class air-gap install ([docs](https://kubesphere.io/zh/docs/v3.3/installing-on-linux/introduction/air-gapped-installation/)) | Open core (QingCloud) | Platform breadth, China strength; not a neutral governed-action primitive; validates air-gap-first distribution in China | +| **Spectro Cloud Palette** | Full-stack cluster lifecycle via Cluster Profiles; VerteX/air-gap editions | Usage-based kilo-Core-hours, edge from ~$250/device/yr ([palette editions](https://www.spectrocloud.com/palette-editions)) | Proprietary, provisioning-centric; not an ops-action governance layer over existing clusters | +| **Devtron** | OSS K8s platform (CI/CD+GitOps+obs+security), v2.1.1 (2026-03-24) *(repo-verified)* | Open core | Batteries-included platform (the breadth strategy), not a narrow federation primitive | +| **Komodor** | SaaS K8s ops/AI-SRE; agent per cluster; Klaudia multi-agent (50+ specialized agents), MCP/OpenAPI extensibility, sandboxed + audited remediation ([press release, 2026-03-18](https://www.globenewswire.com/news-release/2026/03/18/3258257/0/en/komodor-introduces-extensible-autonomous-multi-agent-architecture-for-ai-driven-site-reliability-engineering.html)) | Node-based annual pricing ([pricing page](https://komodor.com/platform/pricing-and-plans/)); closed SaaS | The closest *product* to "operate the fleet with AI on top" — but closed, SaaS (China/air-gap excluded), diagnosis-first; no closed-vocabulary typed intents, no multi-approver canary waves as a neutral primitive | +| **Loft / vCluster** | Virtual clusters for hard multi-tenancy | Open core | Tenancy primitive, not fleet ops; complementary (and a GPU multi-tenancy voice: [vCluster on GPU sharing](https://www.vcluster.com/blog/ai-infrastructure-gpu-utilization-kubernetes-multitenancy)) | +| **kagent** | CNCF agent framework for K8s (v0.10.0-beta4 *(repo-verified)*) | OSS (Solo.io-originated) | Framework to *build* agents — a natural **MCP client of Sith**, not a governance layer | +| **Port / Cortex / Harness** | IDPs / delivery platforms (catalog, scorecards, self-service; Harness = CI/CD+FinOps suite) | Commercial | Different category (the predecessor's failed race). Boundary only: Sith is not a portal; portals can *consume* Sith's API/MCP | + +## 4. Observability / logging (integrate-only; the telemetry-lake trap) + +Prometheus/Grafana/Loki/Mimir, Elastic/OpenSearch/Kibana, Fluentd/Fluent Bit, Datadog, Splunk: each is a query/config surface for Sith to **read through** (per-tool mechanics and auth in [integrations-and-ai-governance.md](integrations-and-ai-governance.md)). The predecessor proved embedding/proxying them is negative value. The fleet-relevant gap none of them fills: they aggregate *telemetry*, not *operational state + actions* — none can answer "which clusters are running image X with a failing rollout and what changed there", and none dispatches governed actions. Datadog/Splunk cost pain is a recurring driver pushing teams toward self-hosted stacks (treated qualitatively here; no pricing figures were fetched in this pass — keep out of load-bearing claims). + +## 5. Cost (workstream F): the fleet rollup is the gap, not the metering + +Fetched evidence chain: + +- **OpenCost** is the CNCF-incubating standard (promoted from Sandbox **2024-10-31**, [CNCF announcement](https://www.cncf.io/blog/2024/10/31/opencost-advances-to-the-cncf-incubator/)) for **per-cluster** allocation: workload-granularity costs from Prometheus + cloud-billing integrations, exported as metrics; the announcement describes no fleet-level aggregation; plugins pull external SaaS costs (Datadog, OpenAI, MongoDB Atlas). +- **The DIY rollup pain is documented first-person by Grafana Labs**: OpenCost "is designed to expect its storage having the same scope as its deployment" — so they run OpenCost per cluster, ship metrics to central Mimir, and had to insert `prom-label-proxy` so each instance sees only its own data; they list native multi-cluster support and multi-cluster query docs as missing ([Grafana Labs on OpenCost](https://grafana.com/blog/2023/02/02/how-grafana-labs-uses-and-contributes-to-opencost-the-open-source-project-for-real-time-cost-monitoring-in-kubernetes/)). A user request for multi-cluster/multi-cloud aggregation was triaged P3 and closed unresolved ([opencost#2638](https://github.com/opencost/opencost/issues/2638)). +- **Kubecost** (the commercial layer; **acquired by IBM, announced 2024-09-17**, [IBM newsroom](https://newsroom.ibm.com/blog-ibm-acquires-kubecost-to-broaden-hybrid-cloud-cost-management-capabilities), folding into the Apptio/Cloudability/Turbonomic FinOps suite): free tier is capped (250 cores, 15-day retention; 3.0's licensing gate ≈ $100k spend/trailing-30-days — [Kubecost 3.0 announcement](https://www.apptio.com/blog/ibm-kubecost-3-0-faster-smarter-and-built-for-scale/)); **"Unified, multi-cluster view" is an Enterprise-tier feature** ([nOps pricing teardown](https://www.nops.io/blog/kubecost-pricing/) — competitor-authored; its $70k–100k/3-cluster anecdote is unverified, treat as directional). 3.0 also moves to a unified IBM agent "eliminating the dependency on Prometheus" — i.e., the leading commercial tool is consolidating into a proprietary suite, not staying a neutral primitive. +- **CAST AI** is automation-first (rightsizing/spot/GPU optimization that *mutates* clusters), quote-based pricing only, and its own intake form lists just EKS/GKE/AKS/OpenShift-on-AWS — **no China clouds, no generic on-prem/air-gap** ([cast.ai/pricing](https://cast.ai/pricing/), fetched 2026-07-08; $108M Series C at ~$900M valuation, [TechCrunch 2025-04-30](https://techcrunch.com/2025/04/30/cast-ai-raises-108m-to-get-the-max-out-of-ai-kubernetes-and-other-workloads/)). +- **GPU:** Kubecost 2.4 does DCGM-based GPU efficiency/idle-cost per container ([Apptio GPU monitoring](https://www.apptio.com/blog/gpu-monitoring/)); MIG/fractional attribution remains beyond it (DCGM reports at physical-GPU level — [vCluster analysis](https://www.vcluster.com/blog/ai-infrastructure-gpu-utilization-kubernetes-multitenancy)); OpenCost GPU pricing has had correctness bugs ([opencost#2029](https://github.com/opencost/opencost/issues/2029)); the FinOps WG GPU paper is single-cluster-only ([FinOps WG](https://www.finops.org/wg/scaling-kubernetes-for-ai-ml-workloads-with-finops/)); Kubecost 3.0's GPU story is one line ("recommendations are GPU-aware"). + +**Verdict for Sith:** cost is a **read-overlay integration, not a wedge and not a build**. The move: deploy/read OpenCost (or its metrics) per cluster via the existing read federation, aggregate at the hub into per-workspace/per-team fleet rollups (GPU columns included where DCGM exists), stamp freshness like every other fleet fact. That lands precisely in the documented OSS gap (per-cluster standard exists; the free fleet view doesn't — it's paywalled at Kubecost Enterprise and unsolved in OpenCost). Building a metering/billing engine or automation-optimizer would re-fight OpenCost/Kubecost/CAST on their ground and violate the telemetry-lake non-goal. + +## 6. Multi-cloud + China clouds (workstream G) + +- **Conformance (hand-verified in [cncf/k8s-conformance](https://github.com/cncf/k8s-conformance)):** Alibaba ACK (`v1.32/alicloud`, "Alibaba Cloud Container Service for Kubernetes v1.32.0"), Huawei CCE (submissions every version v1.29–v1.34), Tencent TKE (`v1.34/tencentcloud`, v1.34.1). The Kubernetes API surface Sith depends on is uniform across US and China clouds. +- **Auth/enumeration is where clouds differ.** EKS: `aws eks list-clusters` + access entries + `aws eks get-token` exec plugin. AKS: `az aks list` + Entra ID + kubelogin. GKE: `gcloud container clusters list` + IAM + `gke-gcloud-auth-plugin` (+ Connect gateway for fleet reach). ACK: kubeconfigs per RAM user/role via `DescribeClusterUserKubeconfig`, configurable expiry/revocation, VPC-internal endpoints ([ACK docs](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/obtain-the-kubeconfig-file-of-a-cluster-and-use-kubectl-to-connect-to-the-cluster)). CCE: IAM for cloud perms + standard kubeconfig for cluster RBAC, intranet/public endpoints ([CCE docs](https://support.huaweicloud.com/intl/en-us/usermanual-cce/cce_10_0187.html)). TKE: separate public/private kubeconfigs, CAM-integrated ([TKE docs](https://www.tencentcloud.com/document/product/457/30639)). +- **OpenShift:** conformant API + `oc`, Routes, SecurityContextConstraints; ACM occupies the fleet layer natively. Treat OpenShift as: conformant-API coverage guaranteed; deep OpenShift-isms (Routes/SCC-aware views) explicitly later; never compete for ACM-committed estates. +- **Recommended abstraction:** exactly two layers. (1) **Pure Kubernetes API** for everything inside a cluster — guaranteed by conformance, including China. (2) **Thin per-cloud adapters** for *enumeration and credential minting only* (list clusters; mint short-lived tokens via each cloud's mechanism; exec-plugin passthrough in local mode). Anything deeper per-cloud (node pools, cloud LBs, billing) is out of scope — that's the cloud console's job. The federated mode needs no cloud adapter at all: the minion dials out, so a TKE cluster behind restricted egress registers exactly like an EKS one (Rancher's ACK/CCE/TKE support proves the pattern; OCM's pull architecture is the same shape). + +## 7. Air-gap distribution prior art + +**Zarf** ([zarf-dev/zarf](https://github.com/zarf-dev/zarf), Apache-2.0, v0.80.0 2026-06-25, ~2k stars, Naval Postgraduate School lineage) owns air-gap *packaging*: single-file bundles of images/charts/manifests, embedded registry/Gitea, image-path rewriting. It is a delivery vehicle, not a control plane — **complement, not competitor**: Sith should ship an official Zarf package (or Zarf-style single artifact) as its China/regulated-market distribution, rather than building bespoke offline tooling. + +## 8. The white space, stated precisely + +Across every tool above, four candidate positions were tested against the evidence: + +1. **Governed action federation as a neutral OSS primitive — EMPTY.** OCM has rollout mechanics but no approvals/typed verbs/audit; ACM/Rancher have platform-coupled policy; Kargo gates artifact promotion only; Komodor audits its own AI's remediation inside a closed SaaS. Nobody ships "typed intent + multi-approver + per-wave gates + abstention + signed dispatch + decision ledger" vendor-neutrally. This is Sith's wedge, confirmed. +2. **Fleet-wide live correlation/search — PARTIALLY OCCUPIED.** Clusterpedia (search) and every platform's inventory exist; correlation joined with actions and staleness semantics does not. Differentiating as part of the wedge, not alone. +3. **Fleet cost read-overlay — EMPTY IN OSS, PAYWALLED COMMERCIALLY.** (§5.) High-value fast-follow on top of read federation. +4. **OSS aggregated multi-cluster local client — EMPTY.** (§1, form-factor file.) The adoption on-ramp. + +And one distribution property that cuts across all four: **air-gap-first, no-phone-home, multi-arch, registry-relocatable** — mandatory for China/regulated (see [Part 1](#part-1)), cheap if designed in from day one, and none of the SaaS incumbents can follow there. + + +--- diff --git a/docs/specs/E2-readfed-brain-integrations.md b/docs/specs/E2-readfed-brain-integrations.md new file mode 100644 index 0000000..fd0a293 --- /dev/null +++ b/docs/specs/E2-readfed-brain-integrations.md @@ -0,0 +1,727 @@ +# Spec — Read Federation, the Investigation Brain, and the Integration Waves + +**Status:** design / spec (ready to slice) · **Date:** 2026-07-10 +**Author role:** Architect C (design/spec only — no product Go code; a builder session implements) +**Epic:** E2 — Read federation ([#20](https://github.com/ArdurAI/sith/issues/20)) · introduces a proposed +new epic **E14 — Investigation Brain** (see §3.7) +**Branch:** `docs/e2-readfed-brain-integrations` off `dev` · **PR target:** `dev` (do **not** merge) + +**Prereqs to read:** [`../SITH-NOTION.md`](../SITH-NOTION.md) (E2, E11, E12, and the roadmap +sections), [`../research/USE-CASE-AND-SHAPE.md`](../research/USE-CASE-AND-SHAPE.md) (the two-wedge +thesis; the connector taxonomy), [`../ARCHITECTURE.md`](../ARCHITECTURE.md) (source-abstract fleet +model; three run-modes). + +**Sits above / beside:** +- **Above** `specs/SLICE-0-foundation.md` — reuses its `fleet.Source`, `FleetResult`, `Cluster`, + `Coverage` types as the load-bearing seam. (That spec and `CONVENTIONS.md` / `BUILD-SEQUENCE.md` + land on `dev` via the parallel lead session's build-plan PR; the cross-references below resolve + once both PRs are on `dev`.) +- **Beside** the **7-verb connector contract** (`discover / read / query / diff / plan / execute / + verify`) that a parallel **Architect A** session specs. This document treats that contract as the + connector seam and *consumes* it; it does **not** redefine it. Where a verb signature is named + here it is a reference to A's seam, and A's spec is authoritative on the wire shape. + +> **Design/spec only.** Every Go snippet below is a **reference shape (design contract)** — types, +> enums, and interface signatures that pin the seam — not implementation. The builder writes idiomatic +> bodies and tests. No function bodies, no product logic here. + +--- + +## 0. TL;DR — the model in five sentences + +1. The normalized fleet model (E2) grows from a flat list of facts into a **four-lens operational + graph**: every entity carries **live** (what is running), **desired** (what should run), + **timeline** (what changed, when), and **telemetry** (how it behaves), correlated by + **OTel-style identity keys** into one graph spanning the whole fleet. +2. Single-cluster tools (k9s / Lens) show **one lens on one cluster**; Sith's differentiator is + **four lenses correlated across the fleet** — which is the substrate root-cause needs. +3. The **Investigation Brain** is a **rule-based, transparent** reasoner over that graph: + `symptom → scope → evidence → hypotheses → validation → plan`, for the failures GR actually hits + (bad deploy, OOMKill, CrashLoopBackOff, config drift, cert expiry, node pressure). +4. The brain **proposes, never executes**: in the **local wedge** its plan is *advisory* (a suggested + command); in the **hub** the same plan becomes a *governed typed intent* through the PEP (E4/E5) — + one brain, two modes, mirroring "one engine, two modes." +5. Integrations arrive in **four waves**; each connector is scored by **which of the 7 verbs it + implements, which lenses it feeds, and an effort tier** — and every brain rule declares the + connectors/lenses it needs, so missing coverage yields honest **abstention** (F2.5), never a + false-confident answer. + +--- + +## 1. Where this fits + +``` + ┌───────────────────────────────────────────────────────────┐ + │ Investigation Brain (E14, proposed) — reasons, proposes │ + │ symptom → scope → evidence → hypotheses → validation → plan │ + └───────────────▲───────────────────────────┬────────────────┘ + │ reads graph │ emits plan + │ ▼ + ┌───────────────┴───────────────┐ advisory (local) │ governed intent (hub, E4/E5) + │ Four-lens Operational Graph │ + │ (E2 extended: live/desired/ │ + │ timeline/telemetry + keys) │ + └───────────────▲───────────────┘ + │ facts (read/query/diff) + ┌───────────────┴───────────────┐ + │ Connectors — 7-verb contract │ ← Architect A's seam + │ (Kubernetes, Argo, Prom, …) │ waves W1–W4 (§4) + └───────────────▲───────────────┘ + │ Source.Fleet(ctx) + ┌───────────────┴───────────────┐ + │ fleet.Source seam (Slice 0) │ local-kubeconfig | ocm-spoke + └────────────────────────────────┘ +``` + +This spec is the **read/reasoning altitude** above Slice-0's foundation and above the connector +verbs. It changes nothing about the write path (E4/E5) except to define the **hand-off point**: the +brain's `plan` output is exactly a typed-intent proposal the existing PEP already knows how to gate. + +--- + +## 2. The model — normalized fleet → four-lens operational graph + +### 2.1 The four evidence lenses + +A "lens" is a way of looking at the same entity. The four are orthogonal and, together, complete +enough to explain the failure modes in §3. Each lens maps to existing fleet-fact kinds plus a small +extension, and to specific connector verbs (§2.5). + +| Lens | Question it answers | Canonical sources | Fleet-fact kind | Retention | +|---|---|---|---|---| +| **LIVE** | *What is running right now?* | K8s API objects + status/conditions, Node conditions, Rollout/Argo live state | `inventory`, `health` | cached current snapshot | +| **DESIRED** | *What is supposed to be running?* | Git manifests, Argo `Application.spec` target, Helm release values, Kustomize render | `desired` (new), `drift` (= live⊖desired) | cached current snapshot | +| **TIMELINE** | *What changed, and when?* | K8s Events, ReplicaSet/Rollout history, Argo sync history, Git commits/PR merges, HPA/autoscaler events, cordon/drain | `change` (new) | **bounded ring** of recent discrete change records | +| **TELEMETRY** | *How is it behaving?* | Prometheus/PromQL, Loki/ES/OpenSearch/Splunk logs, OTel traces, Alertmanager, cert-expiry metrics | `alert` + small derived signals; series stay at source | **query-through** — pulled on demand, not retained | + +**The retention guardrail is load-bearing (SITH-NOTION F2.2).** Sith is *not* a telemetry lake. So: + +- LIVE and DESIRED are bounded **current-state caches** (as today). +- TIMELINE is a **bounded ring of discrete change events** (deploys, syncs, scales, rollouts, drains) + — low-cardinality, event-shaped, aged out — **not** a metric time series. +- TELEMETRY is predominantly **query-through**: the brain queries Prometheus/Loki/etc. *at + investigation time* via the connector `query` verb and retains only the **derived answer** it needs + (e.g. "OOMKilled ×7 in 15m", "p99 rose at 14:03", current alert state). No raw series is stored. A + query-through call may be a **range** query against the *backend's own* retention (that is how R1/R2 + compare before-vs-after a change) — the history lives in Prometheus/Loki, Sith keeps only the + computed verdict. + +This is the single design decision that lets Sith add a telemetry lens **without** becoming the +telemetry store §SCOPE forbids. + +### 2.2 The operational graph + +The graph is the join of the four lenses over a shared entity model. + +- **Nodes** are entities: `Cluster · Namespace · Workload (Deployment/StatefulSet/DaemonSet/Rollout) + · ReplicaSet/Revision · Pod · Container · Node · Image · Service · Ingress · Certificate · PVC · + Application (Argo) · HelmRelease · MeshService`. +- **Edges** are relationships: `runs-on (pod→node) · owned-by (pod→rs→workload) · targets + (Application→workload) · renders-to (HelmRelease/Kustomize→objects) · exposes (Service/Ingress→ + workload) · mounts (pod→PVC) · secured-by (Ingress/MeshService→Certificate) · depends-on + (workload→workload, from mesh/traces) · scheduled-on / deployed-by`. +- Every node carries a **per-lens fact bundle** (some lenses may be empty → coverage gap, §2.6). + +The graph is **fleet-wide**: the same `Image` node can be referenced by workloads in five clusters, +which is exactly how the brain says "this OOMKill is the *same image* on 4 clusters" — a correlation +a single-cluster tool structurally cannot make. + +```mermaid +flowchart LR + IMG["Image sha256:abc"] -- runs-as --> WkA["web@cluster-A"] + IMG -- runs-as --> WkB["web@cluster-B"] + WkA --> RSa["ReplicaSet rev-7"] --> Pa["Pod x3"] -- runs-on --> Na["node-3 (MemPressure)"] + AppA["Argo App web"] -- targets --> WkA + Git["Git repo@rev"] -- desired --> AppA + Prom["Prometheus"] -. query .-> Pa + subgraph lenses["each node carries"] + L1["LIVE"]; L2["DESIRED"]; L3["TIMELINE"]; L4["TELEMETRY"] + end +``` + +### 2.3 Correlation keys — the join (OTel-style) + +Correlation is the hard part: a Prometheus series, a Deployment object, an Argo Application, and a +deploy event must resolve to the **same** logical entity. Sith standardizes on an **`EntityRef`** +built from **OpenTelemetry Kubernetes semantic-convention** attributes, so every connector emits +facts keyed the same way and the graph joins without heuristics. + +| Identity dimension | Key attributes (OTel semconv) | Emitted by | +|---|---|---| +| Cluster | `k8s.cluster.name` (+ Sith `source_kind`, `source_cluster`) | every connector | +| Namespace | `k8s.namespace.name` | K8s, Argo, mesh | +| Workload | `k8s.{deployment,statefulset,daemonset}.name` + `k8s.pod.name` | K8s, Prom, logs, mesh | +| Node | `k8s.node.name` | K8s, Prom, cloud | +| Image | `container.image.name` + **`container.image.digest`** (`sha256:…`) | K8s, registry, Docker | +| Service (telemetry) | `service.name`, `service.namespace` | OTel traces, mesh, Prom | +| Desired unit | Argo `Application` name / Git `repo@path@revision` | Argo, GitHub, Helm | + +Two rules make correlation safe — mis-attribution is worse than a gap, because it would let the +brain blame the wrong entity: + +1. **A fact without a resolvable `EntityRef` is dropped to an "unattached" bucket, never + mis-joined.** +2. **Joins are cluster+namespace-scoped by default.** User-controlled keys (`service.name`, + workload names) collide across teams and clusters — two namespaces can both run a `service.name = + api`. So every join is qualified by `k8s.cluster.name` (+ `k8s.namespace.name`) *first*; a bare + name never joins across that boundary. The **only** genuinely global key is the content-addressed + **`container.image.digest`** (`sha256:…`) — it is safe to join fleet-wide, and it is exactly the + key behind "the same image on 4 clusters." Any other cross-cluster correlation must be an explicit, + named query (e.g. "same Argo `Application` name across clusters"), never an implicit key collision. + +### 2.4 Reference shapes (design contracts) + +These extend Slice-0's `fleet` package. **Reference only** — the builder implements bodies/tests. + +```go +// package fleet — extends SLICE-0-foundation §3.4. + +// Lens is the evidence dimension a fact belongs to. +type Lens string +const ( + LensLive Lens = "live" + LensDesired Lens = "desired" + LensTimeline Lens = "timeline" + LensTelemetry Lens = "telemetry" +) + +// FactKind stays the taxonomy from F2.1, widened for the four lenses. +// inventory|health -> live ; desired|drift -> desired ; +// change -> timeline ; alert (+derived) -> telemetry ; cve -> telemetry/live. +type FactKind string + +// EntityRef is the OTel-semconv correlation key. Zero fields that don't apply. +type EntityRef struct { + Cluster string `json:"k8s.cluster.name"` + Namespace string `json:"k8s.namespace.name,omitempty"` + Kind string `json:"kind,omitempty"` // Deployment, Node, Certificate, ... + Name string `json:"name,omitempty"` + Pod string `json:"k8s.pod.name,omitempty"` + Node string `json:"k8s.node.name,omitempty"` + ImageDigest string `json:"container.image.digest,omitempty"` // sha256:... + Service string `json:"service.name,omitempty"` + App string `json:"argocd.application,omitempty"` +} + +// Fact is one observation on one lens about one entity — freshness/source stamped (F2.2). +type Fact struct { + Ref EntityRef `json:"ref"` + Lens Lens `json:"lens"` + Kind FactKind `json:"kind"` + Payload map[string]any `json:"payload"` // normalized per kind + ObservedAt time.Time `json:"observed_at"` + SourceKind string `json:"source_kind"` // Source.Kind() that produced it +} + +// Graph is the correlated view the brain reads. It never widens Coverage's contract: +// a lens with no fact for an entity is an explicit gap, surfaced (F2.5), never silent. +type Graph interface { + Node(ref EntityRef) (Node, bool) + Neighbors(ref EntityRef, edge EdgeKind) []Node + // Lens returns facts for one lens; ok=false means "not covered" (abstention input). + Lens(ref EntityRef, l Lens) (facts []Fact, ok bool) + Coverage() Coverage // reuses SLICE-0 Coverage, per-lens extended +} +``` + +### 2.5 Lenses ↔ the 7 connector verbs + +The four-lens graph is populated and interrogated through **Architect A's 7-verb contract**. The +mapping is exact and is the contract boundary between this spec and A's: + +| Verb | Purpose | Feeds / uses | Read side or action side | +|---|---|---|---| +| `discover` | enumerate the entities a connector knows | **graph nodes** (all lenses) | read | +| `read` | pull current facts | **LIVE**, **DESIRED** snapshots (cached) | read | +| `query` | on-demand parametric query | **TELEMETRY** (metrics/logs/traces, *not retained*) | read | +| `diff` | live vs desired delta | **DESIRED**/drift lens | read | +| `plan` | propose a change (dry-run/preview) | brain's **plan** output; PEP preview | action (E4/E5) | +| `execute` | apply a change | governed dispatch only | action (E4/E5) | +| `verify` | confirm post-change health | feeds **LIVE/TELEMETRY** back; closes the loop | action (E4/E5) | + +**The brain consumes `discover/read/query/diff`** (the four read verbs) and **emits a `plan`**. It +never calls `execute`. `plan/execute/verify` remain wholly owned by the PEP (E4/E5). This keeps the +"boundaries not instructions" invariant: the reasoner cannot act, only propose. + +### 2.6 Coverage, freshness, abstention (reuse F2.5, do not reinvent) + +Every lens inherits F2.2 freshness stamping and F2.5 staleness → abstention. The extension: coverage +is now **per-lens per-entity**. A brain verdict declares the lenses it required; if a required lens is +stale or absent, the verdict is **downgraded to abstention** with an honest message +("cannot confirm *bad deploy* — timeline lens unavailable for cluster-B"). This is the trust +differentiator over autonomy-first "AI SRE" tools: Sith would rather say *I don't know* than guess. + +--- + +## 3. The Investigation Brain (proposed epic E14) + +### 3.1 What it is, and the stances that define it + +The Brain is a **deterministic, rule-based** reasoner over the four-lens graph that turns a symptom +into a **ranked, evidence-cited set of root-cause hypotheses and a suggested remediation plan**. It is +Sith's differentiator: it is what makes Sith *more than a fleet viewer*. + +Four stances, each a hard requirement: + +1. **Rule-based and transparent, not a black box.** Every hypothesis names the exact signals that + raised it and the evidence that validated it. A reviewer can read *why*. (Contrast HolmesGPT / + Komodor "Liz" LLM-diagnosis — see USE-CASE §1.) +2. **Proposes, never executes.** Output is a *plan*, gated downstream. No autonomy. (The Replit + database-deletion lesson, USE-CASE §6.) +3. **Reads the graph; adds no data source.** The Brain is a reasoning layer; all evidence comes + through the same connector verbs and the same freshness/coverage rules. +4. **Honest about coverage.** Missing lens ⇒ abstain on the dependent hypothesis (§2.6), never a + false-confident verdict. + +### 3.2 The pipeline + +``` +symptom ─▶ scope ─▶ evidence ─▶ hypotheses ─▶ validation ─▶ plan + │ │ │ │ │ │ + trigger walk the gather 4-lens run the confirm/ typed-intent + (alert/ graph from evidence for rule catalog refute each proposal + degraded/ the symptom the scoped (§3.4) → hypothesis (advisory local / + question) node entities candidates w/ targeted governed hub) + (read/query/ ranked by query/diff + diff) weight +``` + +| Stage | Input | Action | Output | +|---|---|---|---| +| **Symptom** | alert / Degraded workload / user question | normalize to `{EntityRef, observed-bad-condition, onset_t}` | `Symptom` | +| **Scope** | `Symptom` | walk graph edges (owned-by, runs-on, targets, depends-on, same-image) to bound the implicated set; detect fleet-wide correlation (same image/config across clusters) | `ScopeSet` (+ `is_correlated`) | +| **Evidence** | `ScopeSet` | pull the four lenses for each entity via `read/query/diff`; stamp freshness; record gaps | `EvidenceBundle` | +| **Hypotheses** | `EvidenceBundle` | evaluate every catalog rule; each matched rule emits a candidate with weight + cited signals | `[]Hypothesis` (ranked) | +| **Validation** | `[]Hypothesis` | for each, gather targeted confirming/refuting evidence; promote/demote; apply coverage gate | `[]Hypothesis` (scored, abstentions marked) | +| **Plan** | top hypothesis | map root cause → suggested remediation as a **typed-intent proposal**; advisory in local, governed in hub | `Plan` | + +### 3.3 Rule schema (reference shape) + +```go +// package brain — reference shape only. + +type RuleID string + +type Signal struct { + Lens fleet.Lens // which lens this evidence lives on + Match string // human-readable predicate, e.g. "container.lastState.reason==OOMKilled" + Weight int // contribution to confidence (+ raises, − refutes) +} + +type Rule struct { + ID RuleID + FailureMode string + Symptom string // the observed-bad condition that makes this rule a candidate + Signals []Signal // evidence patterns (positive and negative) + NeedsLenses []fleet.Lens // coverage gate — absent ⇒ abstain (§2.6) + NeedsVerbs []string // connector verbs required to evaluate (discover/read/query/diff) + RootCause string // the hypothesis statement, templated with entity refs + Plan PlanTemplate // suggested typed intent(s); advisory-rendered in local mode + CauseOf []RuleID // this rule can be the underlying cause of these proximate rules (§3.5) +} + +// PlanTemplate names typed intents from the closed vocabulary (ADR-0004). It is a *proposal*. +type PlanTemplate struct { + Intents []string // e.g. "rollout.undo", "argocd.rollback", "gitops.open-pr", "deployment.scale" + Advisory string // local-mode human command, e.g. "kubectl rollout undo deploy/web -n prod" + Sensitive bool // node/cert/drain actions → always human-approved, never auto-planned +} +``` + +### 3.4 The hypothesis-rule catalog + +The canonical six — the failures GR actually hits. Each is one row expanded. **Signals** are grouped +by lens; **Evidence needed** names the verbs/lenses; **Plan** is a proposal (advisory local / governed +hub). Weights are indicative, to be tuned against real incidents. + +--- + +#### R1 — Bad deploy (regression from a recent change) + +- **Symptom.** Workload Degraded / new pods failing readiness or crashing **shortly after** a change. +- **Signals.** + - *TIMELINE (+3):* a deploy / Argo sync / image-tag change for this workload within ~N min before + `onset_t`. + - *LIVE (+2):* the **new** ReplicaSet/revision is the unhealthy one; the prior revision was healthy. + - *DESIRED (+1):* desired image/tag/manifest changed (Git commit / Argo target advanced). + - *TELEMETRY (+2):* error rate / crash count rose at/after the change timestamp. + - *Refuters (−):* no change in the window; the old revision was already unhealthy; a fleet-wide + infra event coincides (→ R6/node, not deploy). +- **Evidence needed.** rollout history (`read`/LIVE), Argo sync history + Git log (`query`/TIMELINE + + DESIRED), new-RS pod status (`read`/LIVE), error metric before/after (`query`/TELEMETRY). +- **Likely root cause.** The recent change introduced a regression. +- **Suggested plan.** `rollout.undo` **or** `argocd.rollback` to the last-healthy revision. *Local + advisory:* `kubectl rollout undo deploy/ -n `. +- **Coverage gate.** TIMELINE + LIVE mandatory; TELEMETRY strengthens. + +#### R2 — OOMKilled (memory limit / leak / spike) + +- **Symptom.** Container `lastState.terminated.reason == OOMKilled`; restart count climbing. +- **Signals.** + - *LIVE (+3):* `OOMKilled` termination reason; restarts increasing. + - *TELEMETRY (+2):* `container_memory_working_set_bytes` at/near `limits.memory`. + - *TIMELINE (+1):* a recent deploy raised baseline usage, or a limit was lowered — or **none** + (steady leak). + - *DESIRED (context):* the `limits.memory` value in the manifest. + - *Refuters (−):* memory well **below** limit ⇒ node-level OOM/eviction (→ R6), not this rule. +- **Evidence needed.** pod `lastState` (`read`/LIVE), memory-vs-limit series (`query`/TELEMETRY), + limit history + correlated deploy (`query`/TIMELINE, `read`/DESIRED). +- **Likely root cause (disambiguated by telemetry shape).** (a) limit too low for real usage; (b) + **leak** — usage monotonically rises to the limit; (c) **load spike** — usage tracks request rate; + (d) a deploy raised baseline. +- **Suggested plan.** (a/d) bump `limits.memory` via **`gitops.open-pr`** (a *desired-state* change → + PR, **never** a live patch) and/or `deployment.scale` for headroom; (b) flag owner (code fix) + + `rollout.restart` as mitigation; (c) `deployment.scale` / HPA review. *Local advisory:* the same as + a suggested PR diff or `kubectl scale`. +- **Coverage gate.** LIVE confirms the kill; TELEMETRY distinguishes the variant (abstain on the + variant if telemetry absent — still report the OOMKill). + +#### R3 — CrashLoopBackOff + +- **Symptom.** Container waiting reason `CrashLoopBackOff`; ready=false; restarts climbing. +- **Signals.** + - *LIVE (+3):* `CrashLoopBackOff`; container exit code; restart count. + - *TELEMETRY/logs (+3):* last container logs — panic/stack trace, missing config/env, failed + dependency connection, migration failure. + - *DESIRED (+1):* recent config/secret/env change in desired state. + - *TIMELINE (+2):* config change / secret rotation / dependency deploy just before onset. + - *GRAPH (+2):* a `depends-on` neighbour (DB, upstream) is itself unhealthy (→ variant c). + - *Refuters (−):* exit 0 / Completed (Job, not a crash); `ImagePullBackOff` (→ registry/image rule, + §3.5 adjacency). +- **Evidence needed.** waiting reason + exit code (`read`/LIVE), container logs (`query`/TELEMETRY), + config/secret `diff` (DESIRED), dependency health via graph edge (`read`/LIVE + `query`/TELEMETRY). +- **Likely root cause (variants).** (a) app bug from a bad deploy (**cause-of → R1**); (b) + missing/rotated config or secret; (c) failing dependency; (d) mis-set liveness probe killing a + healthy app; (e) bad image/command. +- **Suggested plan.** (a) rollback via R1; (b) fix config via `gitops.open-pr`; (c) **no fleet + action** — point at the failing dependency node; (d) probe fix via `gitops.open-pr`. *Local + advisory* accordingly. +- **Coverage gate.** LIVE + logs(TELEMETRY) are the crux; DESIRED/TIMELINE disambiguate. + +#### R4 — Config drift (live diverged from desired) + +- **Symptom.** Live ≠ desired for a workload/Application (Argo `OutOfSync`, or `diff` verb shows a + delta). +- **Signals.** + - *DESIRED/DIFF (+3):* non-empty `diff` between rendered desired and live; Argo `syncStatus = + OutOfSync`. + - *TIMELINE (+2):* an out-of-band `kubectl edit/scale/patch` event with no matching Git commit + (the drift source); **or** desired advanced but sync disabled/failing. + - *LIVE (context):* the diverging field values. + - *Refuters (−):* diff only in controller-managed/defaulted/status fields ⇒ benign; namespace + excluded from GitOps. +- **Evidence needed.** `diff` verb (DESIRED), Argo sync status + history (`read`/LIVE + `query`/ + TIMELINE), change events (`query`/TIMELINE). +- **Likely root cause (variants).** (a) out-of-band manual mutation; (b) desired advanced but sync is + disabled/failing; (c) a mutating webhook/controller rewrites the field (expected drift). +- **Suggested plan.** (a) `argocd.sync` to reconcile **or** `gitops.open-pr` to capture the live + change into Git if it is the intended one — **abstain** if which state is "true" is ambiguous, and + say so; (b) enable/fix sync then `argocd.sync`; (c) annotate as expected. **Never silently + overwrite.** +- **Coverage gate.** DESIRED + LIVE mandatory (this rule *is* the live-vs-desired join); TIMELINE + attributes the cause. + +#### R5 — Certificate expiry (imminent or expired) + +- **Symptom.** A TLS cert at/near expiry (ingress, webhook, serving, mTLS), or TLS handshake failures. +- **Signals.** + - *LIVE (+3):* cert-manager `Certificate.status.notAfter` within threshold, or not-Ready / + renewal failing; Secret `tls.crt` notAfter near now. + - *TELEMETRY (+2):* `certmanager_certificate_expiration_timestamp_seconds` near now; TLS handshake + errors in logs/metrics; probe failures. + - *TIMELINE (+1):* last renewal event; ACME/issuer order failures. + - *DESIRED (context):* issuer config, `Certificate` spec. + - *Refuters (−):* long validity remaining; TLS errors are SNI/hostname mismatch, not expiry. +- **Evidence needed.** `Certificate`/Secret notAfter (`read`/LIVE), expiration metric (`query`/ + TELEMETRY), renewal/order events (`query`/TIMELINE), issuer health (`read`/LIVE). +- **Likely root cause (variants).** (a) renewal pipeline broken (ACME/DNS/issuer) and cert expiring; + (b) manual cert never automated; (c) clock/issuer misconfig. +- **Suggested plan.** Mostly **advisory / escalation** (sensitive): trigger renewal (e.g. reissue via + cert-manager) or fix issuer config via `gitops.open-pr` — human-approved. High value because it is + **predictable and fleet-wide**: "certs expiring < 7d on N clusters" surfaced *before* the outage. +- **Coverage gate.** LIVE (Certificate/Secret) suffices to **detect**; TELEMETRY confirms impact. + +#### R6 — Node pressure (resource exhaustion / eviction / not-ready) + +- **Symptom.** Pods Pending (unschedulable) or Evicted; Node `MemoryPressure` / `DiskPressure` / + `PIDPressure` / `NotReady`. +- **Signals.** + - *LIVE (+3):* Node conditions True; pod `Evicted`; `FailedScheduling` (Insufficient cpu/memory). + - *TELEMETRY (+2):* node allocatable vs requested; `node_memory_MemAvailable`; disk usage. + - *TIMELINE (+2):* a large deploy/scale-up overcommitted the node; cordon/drain; autoscaler events + (or their **absence** — cluster-autoscaler stuck). + - *GRAPH (+2):* which workloads land on the pressured node (`runs-on` edges) — the blast radius. + - *Refuters (−):* node healthy and Pending is due to affinity/taint/PVC-binding, not capacity (→ + scheduling sub-rule). +- **Evidence needed.** node conditions + allocatable (`read`/LIVE), per-node/pod usage (`query`/ + TELEMETRY), scheduling events (`query`/TIMELINE), autoscaler status (`read`/LIVE). +- **Likely root cause (variants).** (a) genuine capacity shortfall / autoscaler stuck; (b) noisy + neighbour / runaway workload; (c) disk filling (logs/images/PVC); (d) node/kubelet failure + (NotReady). +- **Suggested plan.** (a) scale nodegroup / fix autoscaler — **advisory + escalate** (outside the v1 + verb set; a cloud action later); (b) `deployment.scale` down or `rollout.restart` the noisy + workload; (c) cleanup / PVC expansion — advisory; (d) cordon+drain — **sensitive, human-approved**. + The Brain's value here is **pinpointing cause + blast radius**, not auto-fixing. +- **Coverage gate.** LIVE node conditions mandatory; TELEMETRY quantifies; GRAPH attributes. + +--- + +**Adjacent rules (same pattern, add as coverage lands):** `ImagePullBackOff / registry-auth`, +`FailingDependency` (mesh/trace-driven), `Pipeline/SyncFailure` (Argo/CI), `HPA-thrash`, +`PVC-full / volume-bind`. The schema in §3.3 admits them without change. + +### 3.5 Rule composition and arbitration + +Real incidents fire several rules at once. The Brain resolves them via the graph and the timeline: + +- **Cause-of chaining.** `CauseOf` links a *proximate* symptom to an *underlying* cause. Example: + R3 (CrashLoopBackOff) fires on the pod; R1 (bad deploy) fires on the workload with a config change + at `T0` = the crash onset. The Brain reports **"R3 CrashLoopBackOff — root cause: R1 bad deploy + (config change at 14:02)"** and attaches R1's plan, not R3's generic one. +- **Ranking.** Candidates are ordered by summed signal weight, then by **temporal proximity** to + `onset_t` (a change 30s before beats one 40m before), then by **specificity** (a rule needing more + matched lenses outranks a broad one). +- **Fleet correlation dominates.** If `is_correlated` (same image/config across clusters), the + fleet-wide cause outranks any per-cluster explanation — this is the answer no single-cluster tool + can produce, and the Brain leads with it. +- **Abstention beats a guess.** If the top candidate fails its coverage gate, the Brain reports the + candidate **as unconfirmed** with the missing lens named, and offers the next fully-covered + hypothesis (if any). + +### 3.6 Local advisory brain vs hub governed-plan brain — one brain, two modes + +| | **Local wedge (Phase L)** | **Hub / federation (P1+)** | +|---|---|---| +| Graph source | kubeconfig contexts (client-side fan-out) | OCM spokes (multi-tenant, RLS) | +| Telemetry lens | query-through to **reachable** in-cluster Prom/Loki (port-forward / API proxy); else abstain | connectors run server-side with KMS-enveloped creds | +| Brain output | **advisory** — a suggested command / PR diff the user runs themselves | **governed typed-intent proposal** through the PEP | +| Gating | none (it's the user's own kubeconfig identity) | Ardur PDP + elicited approval + waves + abstention (E4/E5) | +| Execution | the **user** acts (their `kubectl`) | spoke re-validates + executes; `verify` closes the loop | + +The rules, the schema, the pipeline, and the graph are **identical** across modes — only the *plan +rendering* and the *gating* differ. This is the "one engine, two modes" thesis applied to reasoning: +the local wedge ships "k9s for your whole fleet **that also tells you why payments is down**," and the +same brain, in the hub, proposes a governed remediation. + +### 3.7 Where the Brain lives (open decision) + +Two placements, both viable: + +- **(A, recommended) A new epic `E14 — Investigation Brain`**, depending on E2 (graph), E12 + (connectors for coverage), and E4/E5 (the governed-plan hand-off in hub mode). A **local advisory + subset** ships in **Phase L** (rules over the locally-reachable lenses); the **governed-plan** + behaviour lands with the write path. Rationale: the Brain is a distinct capability with its own + acceptance surface and roadmap payoff; an epic makes it a first-class deliverable rather than a + footnote on read-federation. +- **(B) Features `F2.6`+ under E2.** Lighter-weight; keeps "reads + reasoning over reads" together. + Risk: buries the differentiator inside a read epic and muddies E2's tight exit criteria. + +**Recommendation:** open **E14** and add the four-lens graph as **E2 features F2.6 (four-lens +operational graph) + F2.7 (correlation keys / EntityRef)**, so E2 owns the *substrate* and E14 owns +the *reasoning*. Decision deferred to the owner (see §7). + +--- + +## 4. The integration-wave matrix + +### 4.1 Legend + +- **Verbs** — the subset of the 7-verb contract a connector implements (`di`=discover, `rd`=read, + `qy`=query, `df`=diff, `pl`=plan, `ex`=execute, `vf`=verify). +- **Lenses** — which of LIVE / DESIRED / TIMELINE / TELEMETRY it feeds (+ `GRAPH` when it contributes + edges). +- **Kind** — E12/F12.2 taxonomy: **RA** read adapter · **BR** brokered read-through (deep-link, never + re-skin) · **TA** typed-action adapter. +- **Tier** — build effort/complexity: **T0** foundational (core) · **T1** high-value, well-understood + · **T2** moderate (auth/variance) · **T3** specialized / long-tail. +- **Mode** — `local` reachable in the wedge · `hub` server-mode · `both`. + +### 4.2 The matrix + +#### Wave 1 — the daily core (ships with / right behind Phase L) + +| Connector | Kind | Verbs | Lenses | Tier | Mode | Notes | +|---|---|---|---|---|---|---| +| **Kubernetes** (core) | RA + TA | di, rd, qy, df, (pl/ex/vf via actions) | LIVE, TIMELINE (Events/RS history), DESIRED (last-applied), GRAPH | **T0** | both | The substrate — this *is* F2.1/F11.1. Feeds every rule. | +| **GitHub** | RA + TA | di, rd, qy, df, **pl/ex** (`gitops.open-pr`), vf | DESIRED (manifests), TIMELINE (commits/PR/deploys) | **T1** | read=local, write=hub | Read (desired/timeline) local via user token; `gitops.open-pr` is the first governed write (P2). | +| **ArgoCD** | RA + BR + TA | di, rd, qy, **df**, pl/ex (`argocd.sync`,`argocd.rollback`), vf | DESIRED, LIVE, TIMELINE (sync history), drift | **T1** | read=local, sync=hub | Richest single connector — 3 lenses + the exemplar of the `diff` verb. Central to R1, R4. Application CRDs read via kubeconfig. | +| **Prometheus** | RA + query-through | di, **qy**, rd (alerts) | TELEMETRY | **T1** | local-if-reachable / hub | Query-through, not retained. Central to R2, R5, R6 and R1 validation. | +| **Elasticsearch** | RA + query-through | di, **qy** | TELEMETRY (logs) | **T2** | hub (local if creds) | Log search for R3. Auth/index-mapping variance → T2. | +| **AWS** | RA (enum/cred) | di, rd, (qy CloudWatch later) | LIVE (nodes/infra), TIMELINE (CloudTrail later) | **T2** | cluster-enum local; deep facts hub | Enumeration + short-lived token minting (no long-lived keys). Feeds R6 (nodegroup/autoscaler). | + +#### Wave 2 — the desired-state / diff pipeline + +| Connector | Kind | Verbs | Lenses | Tier | Mode | Notes | +|---|---|---|---|---|---|---| +| **Helm** | RA | di, rd, **df**, qy (release history) | DESIRED (rendered), TIMELINE (revisions) | **T2** | local | Release data is in-cluster Secrets → readable via kubeconfig. **Not** an action target in v1 (SCOPE §10). | +| **Kustomize** | RA | rd (render), **df** | DESIRED | **T2** | local | Pure local render of overlays; a desired-state producer, not a live source. Not an action target in v1. | +| **kubectl-diff** | RA (diff helper) | **df** | DESIRED / drift | **T1** | local | The generic `diff` verb (server-side dry-run) for the R4 rule when there is no Argo. Thin. | + +#### Wave 3 — visualization, tracing, more clouds + +| Connector | Kind | Verbs | Lenses | Tier | Mode | Notes | +|---|---|---|---|---|---|---| +| **Grafana** | **BR** | di, (deep-link) | TELEMETRY (via deep-link) | **T2** | hub | Brokered read-through **only** — deep-links to Grafana's own UI, never re-skins (E12/F12.2). Underlying series come from Prom/Loki. | +| **OTel** | RA + key backbone | rd (resource attrs), **qy** (traces) | TELEMETRY (traces), **+ correlation-key reference** (§2.3) | **T3** | hub | Special: the semconv backbone for `EntityRef`. Trace query feeds R3 dependency edges. | +| **OpenShift** | RA + TA | di, rd, qy, df | LIVE, TIMELINE, DESIRED (+ Routes/DC/SCC) | **T2** | both | K8s superset; conformant API works day-1, OpenShift objects add surface. | +| **Azure** | RA (enum/cred) | di, rd | LIVE (AKS/infra), TELEMETRY (Azure Monitor later) | **T2** | cluster-enum local; deep facts hub | AKS enum + Entra/kubelogin token minting. | +| **GCP** | RA (enum/cred) | di, rd | LIVE (GKE/infra), TELEMETRY (Cloud Monitoring later) | **T2** | cluster-enum local; deep facts hub | GKE enum + gke-gcloud-auth-plugin. | + +#### Wave 4 — long-tail logs, mesh, containers + +| Connector | Kind | Verbs | Lenses | Tier | Mode | Notes | +|---|---|---|---|---|---|---| +| **OpenSearch** | RA + query-through | di, **qy** | TELEMETRY (logs) | **T3** | hub | ES-fork parity; mostly reuses the ES connector shape. Feeds R3. | +| **Splunk** | RA + query-through | di, **qy** | TELEMETRY (logs/events) | **T3** | hub | SPL query; auth complexity → T3. Feeds R3, some TIMELINE via events. | +| **Fluentd / FluentBit** | RA (**health only**) | di, rd | **LIVE** (pipeline health) — **not** a telemetry source | **T3** | both | **Scope discipline:** Sith reads the log **sinks** (ES/OpenSearch/Splunk/Loki), and treats the shippers only as monitored workloads. Never ingests through them (SCOPE §10). | +| **Istio / Linkerd** | RA | di (mesh services → **GRAPH edges**), rd (mTLS/proxy status), qy (golden metrics) | TELEMETRY (mesh), LIVE (mTLS), **GRAPH (depends-on edges)**, feeds R5 (mTLS certs) | **T3** | hub (local if reachable) | The `depends-on` edges power R3 dependency-walk and scope correlation. | +| **Docker** | RA | di, rd, qy | LIVE (local containers), **image-digest keys** | **T3** | local | Peripheral to the fleet story: local-dev container view + image-digest correlation keys. | + +### 4.3 Connector → brain-rule coverage map + +The Brain's honesty depends on this: a rule can only reach a **confident** verdict when its required +lenses are covered by an installed connector. Otherwise it **abstains** (§2.6, §3.5). + +| Rule | LIVE | DESIRED | TIMELINE | TELEMETRY | Minimum connectors for a confident verdict | +|---|---|---|---|---|---| +| R1 bad deploy | K8s | Argo/GitHub/Helm | K8s Events + Argo/GitHub | Prom (validation) | **Kubernetes + (Argo or GitHub)**; Prom strengthens | +| R2 OOMKilled | K8s | K8s/Helm | K8s/Argo | **Prometheus** (variant) | **Kubernetes** (detect) **+ Prometheus** (variant) | +| R3 CrashLoop | K8s | Argo/GitHub | K8s/Argo | **logs** (ES/OpenSearch/Splunk/Loki) | **Kubernetes + a log connector**; mesh for dependency variant | +| R4 config drift | K8s | **Argo or kubectl-diff/Helm/Kustomize** | K8s Events | — | **Kubernetes + (Argo or kubectl-diff)** | +| R5 cert expiry | K8s (cert-manager) | Argo/GitHub | K8s Events | Prom (impact) | **Kubernetes/cert-manager**; Prom confirms impact | +| R6 node pressure | K8s (Node) | — | K8s Events + cloud autoscaler | **Prometheus** (quantify) | **Kubernetes** (detect) **+ Prometheus** (quantify) + cloud (autoscaler) | + +**Reading the map:** with only the **Wave-1 core (Kubernetes + Argo + GitHub + Prometheus + a log +store)**, all six rules reach at least a *detect* verdict, and R1/R2/R4/R5/R6 reach *confident*. R3's +confidence needs a log connector. This is why W1 is the daily core — it is precisely the coverage the +six rules need. + +### 4.4 Scope-discipline call-outs (anti-drift, from SCOPE §10) + +- **Fluentd/FluentBit are not data sources** — read the sinks; treat shippers as health-only. +- **Grafana is deep-link only** — brokered read-through, never re-skinned (no iframe-Grafana trap). +- **Helm/Kustomize are not action targets in v1** — desired-state *readers*, not mutation surfaces. +- **No telemetry retention** — TELEMETRY is query-through; only derived answers are kept (§2.1). +- **No `exec` / free-form apply** in any plan — the closed vocabulary (ADR-0004) bounds every + `plan`; node/cert/drain remediations are `Sensitive` → always human-approved. + +--- + +## 5. The local wedge / hub line (consolidated) + +| Capability | **Local wedge (Phase L, day-0)** | **Hub / federation (P1 → P3)** | +|---|---|---| +| Fleet source | kubeconfig contexts (client-side fan-out) | OCM spokes, cross-VPC/NAT, multi-tenant + RLS | +| LIVE lens | ✅ kubeconfig | ✅ OCM-brokered read | +| DESIRED lens | ✅ in-cluster CRDs (Argo Application, Helm Secrets), Kustomize/kubectl-diff render, Git read (user token) | ✅ same, server-side | +| TIMELINE lens | ✅ K8s Events + RS/rollout history + Argo sync history + Git log | ✅ same + federated | +| TELEMETRY lens | ⚠️ query-through to **reachable** in-cluster Prom/Loki; else abstain | ✅ server-side connectors, KMS-enveloped creds | +| Correlation / cross-cluster query | ✅ across all local contexts (F11.4) | ✅ across the workspace's spokes (F2.3) | +| **Investigation Brain** | ✅ **advisory** — hypotheses + suggested command/PR (E14 local subset) | ✅ **governed** — plan → PEP → approval → dispatch → `verify` | +| Governance | none — user's own identity | Ardur PDP, elicited approval, waves, abstention, audit + decision-ledger | +| Trust posture | no account, no telemetry-egress, nothing leaves the machine | shared audit, multi-approver prod, tenant isolation | + +**The one seam that makes this work:** the four-lens graph and the Brain are **source-abstract and +mode-abstract**. Local mode is the graph with a kubeconfig source and an advisory plan renderer; hub +mode is the same graph with an OCM source and a governed plan renderer. Nothing above the source and +the plan-renderer forks. + +--- + +## 6. Acceptance criteria + +### 6.1 Four-lens operational graph (E2 F2.6/F2.7) + +- [ ] Facts carry a `Lens` and a resolvable `EntityRef` (OTel semconv keys); a fact with no resolvable + ref goes to an explicit *unattached* bucket, never mis-joined. +- [ ] For one workload, the graph returns non-empty **LIVE** and **DESIRED** bundles and a **`diff`** + (live⊖desired) that matches an independent `kubectl diff` / Argo `OutOfSync`. +- [ ] The **TIMELINE** lens returns the recent change events for a workload in order, as a **bounded + ring** (old events aged out) — verified to hold **no** metric time series. +- [ ] The **TELEMETRY** lens answers a bounded query (e.g. memory-vs-limit) **query-through** and + retains only the derived answer — verified that no raw series is persisted. +- [ ] Per-lens **coverage** is reported; a stale or absent lens is surfaced (F2.5), never silent. +- [ ] The **same** `Graph` interface is satisfied by a local-kubeconfig source and by a second + in-memory/OCM-shaped source (the source-abstraction guarantee), proven in tests. +- [ ] A fleet-wide correlation (same `image.digest` across ≥ 2 clusters) is expressible as one query + and returns the matching clusters with staleness flagged. + +### 6.2 Investigation Brain (E14) + +- [ ] The pipeline `symptom → scope → evidence → hypotheses → validation → plan` runs end-to-end over + the graph for at least the six catalog rules (R1–R6). +- [ ] Each of R1–R6 produces, on a seeded fixture, the **correct top hypothesis** with its **cited + signals** and a **suggested plan** naming only closed-vocabulary intents (or an advisory + command in local mode). +- [ ] **Cause-of chaining** works: a CrashLoop caused by a bad deploy reports R1 as the root cause of + R3 and attaches R1's plan. +- [ ] **Coverage gate / abstention:** with a required lens made stale/absent, the dependent hypothesis + is reported **unconfirmed** with the missing lens named — never a false-confident verdict. +- [ ] **Fleet correlation dominates:** when the same cause spans ≥ 2 clusters, the fleet-wide + hypothesis outranks any per-cluster explanation. +- [ ] The Brain **never** calls `execute` — its only output is a `plan` (advisory or governed); + verified by the seam (no execute code path in `brain`). +- [ ] **Local vs hub:** the identical rule set yields an *advisory command* in local mode and a + *typed-intent proposal routed to the PEP* in hub mode — same hypotheses, different plan renderer. + +### 6.3 Integration-wave matrix + +- [ ] Every connector in §4.2 declares its **kind** (RA/BR/TA), its **verb subset**, and the + **lenses** it feeds; the framework rejects an out-of-taxonomy connector (E12/F12.2). +- [ ] With **only the Wave-1 core** installed, R1/R2/R4/R5/R6 reach a *confident* verdict and R3 + reaches at least *detect* — matching the coverage map (§4.3). +- [ ] **Scope discipline holds:** Fluentd/FluentBit expose LIVE health only (no log ingestion through + them); Grafana is deep-link only; Helm/Kustomize expose no action verbs in v1. +- [ ] Each connector's **mode** (local / hub / both) is honored — a local-only run uses no + server-side connector and no stored credential. +- [ ] Effort tiers are recorded so the build order (T0 → T1 → …) is unambiguous. + +--- + +## 7. Dependencies, open decisions, references + +### 7.1 Dependencies + +- **Slice-0 foundation** (`specs/SLICE-0-foundation.md`, arriving on `dev` via the build-plan PR) — + provides `fleet.Source`, `FleetResult`, `Cluster`, `Coverage`. This spec extends that package. +- **Architect A's 7-verb connector contract** — authoritative on the wire shape of + `discover/read/query/diff/plan/execute/verify`. §2.5 and §4 consume it; if A's verb names/signatures + differ, A's spec wins and this doc is reconciled. +- **E4/E5 (action + policy federation)** — own `plan/execute/verify` and the PEP. The Brain's hub-mode + plan is a typed-intent proposal handed to that pipeline; this spec adds no new write surface. +- **E12 (connector framework)** — the RA/BR/TA taxonomy and one-canonical-connector rule that §4 + scores against. +- **F2.5 (staleness/abstention)** — reused wholesale for per-lens coverage and Brain abstention. + +### 7.2 Open decisions (for the owner) + +1. **Brain placement — E14 epic vs E2 features.** Recommendation: **E14** for the reasoner + **E2 + F2.6/F2.7** for the graph substrate (§3.7). Needs an owner call and, if accepted, a new GitHub + issue for E14 and two feature issues under E2 (#20). +2. **Local telemetry reachability.** How aggressively should the local wedge auto-port-forward to + in-cluster Prometheus/Loki vs. requiring an explicit endpoint? Affects how often R2/R6 can reach a + *confident* (not just *detect*) verdict locally. Recommendation: opt-in endpoint + best-effort + API-proxy discovery; abstain cleanly when absent. +3. **Timeline ring size / retention window.** The bounded-ring depth (events and age) that keeps + "what changed recently" useful without drifting toward a store. Recommendation: a small fixed + window (e.g. last N events / last 24–72h), tuned against real incidents; explicitly *not* a series. +4. **Rule weights and thresholds.** The §3.4 weights are indicative. They should be tuned against a + corpus of GR's real incidents; consider a fixtures/replay harness so rule changes are regression- + tested. +5. **Cross-cluster remediation surface for node/cluster-level causes (R6).** Node/nodegroup/autoscaler + actions are outside the v1 closed vocabulary — kept advisory now. Whether/when to add a + cloud-scoped typed verb is a later E4 decision. +6. **`EntityRef` canonicalization.** Confirm OTel semconv attribute names as the canonical keys (vs a + Sith-internal scheme). Recommendation: OTel semconv, so connectors and traces align for free. + +### 7.3 References + +- Epics & features: `SITH-NOTION.md` — E2 (read federation), E11 (local wedge), E12 (connector + framework), E13 (cost overlay), E4/E5 (action/policy), F2.1–F2.5. +- Roadmap: `ROADMAP.md` — Milestone-0, Phase L, P1–P3. +- Thesis & taxonomy: `research/USE-CASE-AND-SHAPE.md` — two wedges; three connector kinds; the day-1 + six; the "boundaries not instructions" lesson. +- Architecture: `ARCHITECTURE.md` — source-abstract fleet model; three run-modes; the PEP. +- ADRs: `adr/0004-typed-intent-action-model.md` (closed vocabulary), `adr/0005-ai-mcp-ardur-pdp.md` + (PDP), `adr/0006-credential-key-custody.md` (custody). +- External: OpenTelemetry Kubernetes semantic conventions (correlation keys, §2.3). diff --git a/docs/specs/F11.2-F11.5-local-fleet-ux.md b/docs/specs/F11.2-F11.5-local-fleet-ux.md new file mode 100644 index 0000000..2286a1f --- /dev/null +++ b/docs/specs/F11.2-F11.5-local-fleet-ux.md @@ -0,0 +1,621 @@ +# F11.2 + F11.5 — Local Fleet UX (cache-first fleet render + per-pod table stakes) + +**Status:** draft for review · **Date:** 2026-07-10 · **Author:** Architect B (design/spec) +**Scope:** the day-0 local adoption experience — the `sith` CLI + TUI cache-first fleet render +(**F11.2 / #33**) and per-pod table stakes (**F11.5 / #35**). Maps to **Slice 2** and **Slice 3** +of `BUILD-SEQUENCE.md`. +**Product identity:** ArdurAI. **Mode:** local, offline, loopback-only, no account, no telemetry. + +> This is a **design/spec** document. It defines information architecture, screens, states, +> keybindings, the cache-first render model, the resource-lens ordering, and acceptance criteria. +> It contains **no product Go code** — a builder session implements against it. Type names and +> command signatures shown are contracts to build to, not source to copy. + +--- + +## 0. How this spec sits in the plan + +| Layer | Owner | This spec's relationship | +|---|---|---| +| `fleet.Source` seam + `FleetResult`/`Cluster`/`Coverage` types | Slice 0 (`SLICE-0-foundation.md`) | Consumed as-is; this UX renders whatever the model exposes. | +| Local-kubeconfig adapter + client-side fan-out (F2.1 #38 / F11.1 #32) | Slice 1 | The source of facts this UX renders; unreachable/staleness semantics come from here. | +| **7-verb source-adapter contract** (discover / read / query for reads; streaming/session verbs for the rest) | **Architect A (parallel)** | **Referenced, not redefined.** §6 maps every per-pod action onto A's verbs. Where a verb's exact name is A's to fix, this spec names the capability and flags the binding as an open decision (§11). | +| **Cache-first fleet render (CLI + TUI)** — F11.2 #33 | **This spec** (Slice 2) | §§3–5, 7–10. | +| **Per-pod table stakes** — F11.5 #35 | **This spec** (Slice 3) | §6. | +| Local web "fleet IDE" `sith ui` (F11.3 #34) | Slice 4 | Out of scope here; §12 notes the invariants the web surface must mirror so it stays "one engine, two faces". | + +**Non-negotiable invariants inherited from `CONVENTIONS.md` §7** (every screen and action in this +spec obeys them): loopback-only; no account / no telemetry; credentials never leave the machine; +per-pod ops use the **user's own kubeconfig identity** and are **never** governed typed intents; +fail-safe never fail-open (stale/partial view is surfaced, never silently completed). + +--- + +## 1. The promise this UX has to keep + +> `brew install sith && sith` → every kubeconfig context auto-detected → an aggregated fleet view +> with cross-cluster search, **in under 10 minutes, offline, nothing leaving the machine.** +> (E11 exit criteria, `SITH-NOTION.md`.) + +The reference user **GR** lives in `k9s` and Lens daily, across multi-cluster Kubernetes on +AWS/Azure/GCP, all kubeconfig-based. GR already has muscle memory. The bar is not "teach GR a new +tool" — it is **"be the tool GR already knows, but for the whole fleet at once."** Two design +consequences fall straight out: + +1. **k9s-informed, not k9s-cloned.** Where k9s has a good idiom (`:`-command, `/`-filter, single-key + resource nav, resource-scoped actions), inherit it verbatim so GR is productive in minute one. + Where k9s is *structurally single-cluster* (one context at a time, context-switch is a modal + change), **replace it** with a fleet-native model: the cluster is a *column*, not a *mode*. +2. **The centre of gravity is the fleet, not the pod.** This is not another single-cluster console + that happens to loop over contexts. The default landing view is cross-cluster. Per-pod actions + (§6) are table stakes so the tool is *complete* — they are the floor, not the differentiator. + +--- + +## 2. Design principles (the five that decide every trade-off) + +1. **Cache-first, always.** The local store is the *single render source*. No view, keystroke, + sort, filter, or navigation ever blocks on the Kubernetes API. The API is a background sync + target only. Spinner-first is a bug (§3). +2. **Honest coverage over false completeness.** A fleet answer that silently dropped an unreachable + cluster is worse than one that says "9 of 11 clusters — 2 unreachable." Every aggregate view and + every search result carries a coverage line. Staleness is shown, never hidden (§4). +3. **Fleet is a dimension, not a mode.** `CLUSTER` is a first-class column and a first-class filter. + You never "switch into" a cluster to see it; you *scope* to it. Scoping is reversible and + non-modal. +4. **Local identity, local blast radius.** Every per-pod action runs as the user's own kubeconfig + identity against the selected context — exactly what their `kubectl` would do. Nothing here is a + Sith-brokered or governed action; there is no PEP, no intent, no approval in local mode (§6). +5. **Boringly fast, quietly honest.** Sub-100 ms renders, no gratuitous animation, no chrome that + costs a row of `pods`. Trust is won by the tool *feeling* like it respects the terminal and the + machine. + +--- + +## 3. Cache-first render model (the heart of F11.2) + +### 3.1 The store is the only thing the UI reads + +``` + Per-context watch/read sessions (Slice 1, F2.1) The render path (this spec) + ┌───────────────────────────────┐ ┌──────────────────────────────┐ + │ ctx: prod-eu ─ watch stream ─┐ │ TUI view ─ reads store only │ + │ ctx: prod-us ─ watch stream ─┼──► ┌───────────────┐ │ CLI verb ─ reads store only │ + │ ctx: staging ─ watch stream ─┼──► │ LOCAL FLEET │──►│ command bar─ reads store only │ + │ ctx: dev-az ─ watch stream ─┘ │ STORE │ │ search ─ reads store only │ + │ ctx: lab-gcp ─ (unreachable) ──────►│ (single src) │ └──────────────────────────────┘ + └───────────────────────────────┘ └───────────────┘ ▲ + ▲ │ │ keystroke / query + └── background delta ──────┘ │ (never hits API) + reconcile (async) user ────────┘ +``` + +**Rule (testable):** on any keystroke, sort, filter, lens change, or navigation, the render reads +the in-memory store and returns in **< 100 ms** (p95, warm store). The API is touched only by the +background sync loop, never on the interaction path. This is the F11.2 acceptance bar and the +Slice-2 parity requirement. + +### 3.2 Freshness is per-cell, per-cluster, and per-view + +The store stamps every fact with `observed_at` and `source_cluster` (Slice-0 `Cluster.ObservedAt`, +F2.2). The UX surfaces freshness at three granularities: + +| Granularity | Where shown | Signal | +|---|---|---| +| **Per-cluster** | Cluster column / cluster bar | `age` since last successful sync; color ramps green→amber→red as it crosses thresholds. | +| **Per-view (coverage)** | Status/footer line of every aggregate view | `covered N/M clusters · K stale · J unreachable` — the honest-coverage line (§4). | +| **Per-row (optional)** | A dim `⟳` glyph on rows whose owning cluster is mid-resync | Reconcile in flight; row may update momentarily. | + +**Staleness thresholds** (defaults, configurable in `~/.config/sith/config.yaml`): + +| State | Meaning | Default threshold | Visual | +|---|---|---|---| +| `fresh` | last sync within window | ≤ 15 s | no marker (green cluster dot) | +| `stale` | data aging, still shown | 15 s – 2 min | amber age, `~` prefix on the cluster | +| `very-stale` | likely drifted | > 2 min | red age, `!` prefix, row dimmed | +| `unreachable` | no session / session dropped | n/a | red `✗`, cluster excluded from aggregates *and counted in coverage* | + +**Never** blank a stale value: show the last-known value **with** its staleness marker. A blank +cell reads as "zero"; a stale-marked value reads as "last known, N ago" — which is the truth. + +### 3.3 States of the store over a session + +| State | When | What the UI shows | What it must NOT do | +|---|---|---|---| +| **Cold** | first launch, store empty | Skeleton table with header + `warming N contexts…` per-cluster progress; whatever has landed renders immediately | Block the whole screen on a spinner; wait for the slowest context | +| **Warming** | some contexts hydrated, some pending | Rendered rows for ready clusters; pending clusters show `warming` in the cluster bar with a coverage line `covered 3/11` | Hide the ready data behind the pending data | +| **Warm** | all reachable contexts hydrated | Full render, `<100 ms` interactions, background reconcile | Re-fetch on keystroke | +| **Degraded** | ≥1 context dropped mid-session | Last-known data for the dropped cluster, marked `very-stale`→`unreachable`; coverage line updates; a one-line toast: `lab-gcp unreachable — showing last-known (2m ago)` | Silently drop the cluster from aggregates | +| **Offline** | machine has no network / VPN down | Entire fleet ages into `unreachable`; store still renders last-known; banner: `offline — showing last-known fleet (oldest 4m)` | Error out; clear the screen; imply the data is live | + +**Cold-start budget:** the *first paint* (header + skeleton + any already-hydrated cluster) must +happen in **< 250 ms** regardless of how many contexts exist or how slow the slowest is. Fan-out is +non-blocking (Slice 1 guarantee); the UI paints the frame and fills it as facts arrive. + +### 3.4 Background refresh & delta reconciliation + +- Watch streams push deltas into the store continuously; the UI observes the store and repaints + changed rows only (no full-screen redraw storms). +- A **visible, non-modal** reconcile indicator lives in the status line: a single spinner glyph + `⟳` next to the cluster count while any background sync is in flight. It never steals focus and + never blocks input. +- **Manual refresh** (`Ctrl-R` / `:refresh`) forces a resync of the current scope; it is a *hint to + the background loop*, not a synchronous fetch — the UI stays interactive while it runs. +- **Pause/resume watches** (`:pause`) freezes the store for a stable read (useful when triaging a + fast-churning namespace); the footer shows `PAUSED — data frozen 00:42`. + +--- + +## 4. Honest coverage & staleness (the trust surface) + +Every **aggregate** surface (any view that spans > 1 cluster, and every search/correlation result) +ends with a **coverage line**. This reuses the Slice-0 `Coverage{Requested, Reachable, Unreachable}` +and F2.5 staleness semantics — the UX just renders them consistently. + +``` + covered 9/11 clusters · 1 stale (prod-eu, 3m) · 1 unreachable (lab-gcp) [c] coverage +``` + +- `covered N/M` — how many of the scoped clusters answered. +- `stale (…)` — clusters whose newest fact is past `stale` threshold, with the oldest age. +- `unreachable (…)` — clusters with no live session; **counted, named, never omitted.** +- `[c] coverage` — pressing `c` opens the **coverage popover**: a per-cluster table of + `reachable · last-sync-age · error (if any)`, so the gap is one keystroke from being understood. + +**Design law:** if the coverage line is not `covered M/M · 0 stale · 0 unreachable`, the result is +**not** presented as complete. A cross-cluster search that couldn't reach a cluster says so *in the +result*, not in a log the user won't read. (This is the F11.4 / F2.5 guardrail rendered.) + +--- + +## 5. Information architecture + +### 5.1 The mental model (one hierarchy, fleet at the root) + +``` + FLEET ──► CLUSTER (context) ──► NAMESPACE ──► RESOURCE KIND ──► OBJECT ──► (pod actions §6) + ▲ ▲ ▲ ▲ + │ │ │ └─ the "lens" (pods, deployments, …) + │ │ └─ ns scope: all / one / a set + │ └─ cluster scope: all / one / a labeled set (e.g. env=prod) + └─ default landing scope: ALL clusters, ALL namespaces, PODS lens +``` + +Scope is a **filter stack**, not a navigation stack. You are always looking at *one lens* through +*one cluster-scope* and *one namespace-scope*. Changing any of the three is instant and reversible. +This is the key departure from k9s: **cluster is a filter dimension, never a modal context you drop +into and out of.** + +### 5.2 CLI information architecture + +The CLI is the scriptable, SSH-friendly, pipe-friendly face of the *same store*. Parity with the +TUI is a hard acceptance criterion (F11.2): a `--all-clusters` CLI verb returns the same aggregated +answer the TUI shows. + +**Primary read verbs** (render from cache; `-o json|yaml|wide|name` for scripting): + +| Command | Does | Notes | +|---|---|---| +| `sith clusters` | list contexts + reachability + last-sync age | Slice-0 shape, now real (Slice 1). Coverage-honest. | +| `sith get [name] [-A\|-n ns] [--all-clusters]` | fleet-wide or scoped resource list | `` = pods, deploy, events, nodes, svc, … (§5.5). `--all-clusters` is the fleet verb; `--context ` scopes to one. | +| `sith search ` | cross-cluster fuzzy + structured search (§7) | Prints matches with a `CLUSTER` column + coverage line. | +| `sith correlate ` | the signature cross-cluster query (§7.3) | e.g. `sith correlate 'deploy/payments status!=Healthy'`. | +| `sith describe / --context [-n ns]` | human-readable object + events | Maps to read×2 (§6.4). | +| `sith yaml / --context [-n ns]` | raw object YAML to stdout | Maps to read (§6.5). Pipe-friendly. | + +**Per-pod action verbs** (Slice 3, §6 — user's own identity, one explicit context): + +| Command | Does | +|---|---| +| `sith logs --context [-n ns] [-c container] [-f] [--tail N] [--since D]` | stream/tail logs | +| `sith exec --context [-n ns] [-c container] -- ` | exec (interactive with `-it`) | +| `sith port-forward / --context [-n ns] LOCAL:REMOTE` | port-forward | + +**CLI design rules:** +- **Fleet verbs are opt-in explicit** (`--all-clusters`) so `sith get pods` in a scripting context + is never accidentally a 40-cluster fan-out; but the **TUI default scope is all-clusters** (the + wedge's whole point). The asymmetry is deliberate: interactive = fleet-first; scripted = explicit. +- **Action verbs require an explicit `--context`.** There is no "current cluster" for a mutating-ish + action; you name the cluster you're exec-ing into. (Reads may fan out; a shell into a pod may not + be ambiguous about *which* pod.) +- Every read verb honors `-o json` with a **stable schema** (the `FleetResult`/fact shape) so GR can + `| jq`. Coverage is a field in JSON (`.coverage`), a footer line in text. +- Exit codes: `0` complete, `0` with a warning to stderr if partial-but-answered, non-zero only on + *usage* error or *total* failure (no reachable cluster). Partial coverage is **not** a failure — + it's a surfaced fact. + +### 5.3 TUI information architecture — screen regions + +``` +┌─ HEADER ──────────────────────────────────────────────────────────────────────────────────┐ +│ sith ⎈ fleet: 11 ctx (9✓ 1~ 1✗) scope: all-clusters · all-ns lens: Pods ⟳ │ +│ context menu ▸ [0] all [1] prod-eu [2] prod-us [3] staging … │ +├─ BREADCRUMB / SCOPE BAR ────────────────────────────────────────────────────────────────────┤ +│ Pods · all-clusters · all-namespaces · filter:(none) · sort:CLUSTER↑ │ +├─ MAIN TABLE ───────────────────────────────────────────────────────────────────────────────┤ +│ CLUSTER NAMESPACE NAME READY STATUS RESTARTS AGE NODE ⟳ │ +│ prod-eu payments payments-7d9c-abcde 1/1 Running 0 4d n-12 │ +│ prod-us payments payments-55f8-fghij 0/1 CrashLoop 7 (2m) 1h n-03 ⟳ │ +│ staging web web-6b4-klmno 2/2 Running 0 9d n-21 │ +│ ~prod-eu ... (last-known 3m ago, resyncing) ⟳ │ +│ ✗lab-gcp — unreachable — showing nothing for this cluster (see coverage) — │ +├─ FOOTER / STATUS ────────────────────────────────────────────────────────────────────────────┤ +│ covered 9/11 · 1 stale · 1 unreachable [c]overage [/]filter [:]cmd [?]help 3 selected │ +└──────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Regions:** +- **Header** — identity + fleet summary (`N ctx (reachable✓ stale~ unreachable✗)`) + current scope + + current lens + reconcile glyph. The context strip gives single-key cluster scoping (`0`=all, + `1..9`=named clusters, `:ctx ` for the rest). Cloud badges (`aws`/`az`/`gcp`) come from the + exec-plugin detected in Slice 1 and let GR read the fleet's cloud mix at a glance. +- **Scope bar** — the current filter stack (lens · cluster-scope · ns-scope · text filter · sort), + always visible so the user is never confused about *why* they see these rows. +- **Main table** — the lens, rendered from cache. Cluster is the leftmost column in all-cluster + scope (and hidden when scoped to one cluster, to reclaim width). Stale/unreachable clusters are + visually marked inline (`~`/`✗` prefixes) and also summarized in the footer. +- **Footer** — coverage line (§4) + the always-there keyhint strip + selection count. + +### 5.4 Navigation model (k9s idioms, fleet-extended) + +| Idiom | k9s behavior | sith behavior (fleet-extended) | +|---|---|---| +| `:` command | jump to a resource (`:pods`, `:deploy`) | same, **plus** fleet scope words: `:pods !prod` (all but prod), `:deploy @env=prod` | +| `/` filter | fuzzy-filter the current table | same; filters across **all shown clusters** at once | +| Enter | drill into resource | drill: pod → containers/logs; deploy → its pods (fleet-wide); node → its pods | +| Esc | up one level | pop the drill; at root, clears filter | +| number keys | (n/a) | `1..9` scope to the Nth cluster; `0` back to all-clusters | +| `<` / `>` | (n/a) | cycle cluster-scope through the labeled cluster sets | +| `s` | (varies) | shell/exec into selected pod (§6.2) | +| `l` | logs | logs for selected pod (§6.1) | +| `d` | describe | describe selected object (§6.4) | +| `y` | YAML | view/edit YAML of selected object (§6.5) | +| `f` | port-forward | port-forward selected pod/svc (§6.3) | + +The single most important extension: **every list is cross-cluster by default, and every filter, +sort, and search spans the whole fleet** — the thing k9s users asked for twice and were told +"not planned" (`SITH-NOTION.md` §wedge, k9s #1006 / #2730). + +### 5.5 Resource lenses — ordered to daily K8s ops + +The lens order is deliberately **frequency-ranked to GR's actual day**, not alphabetized and not a +completeness parade. Tier 1 is the minute-to-minute loop; it ships in Slice 2. Tiers 2–3 are the +same render code over more kinds and land in the same slice or immediately after (they cost columns, +not architecture). + +**Tier 1 — the daily loop (must be first, must be excellent):** + +| Lens | Why it leads | Fleet-specific value | +|---|---|---| +| **Pods** | the default landing view; where "is it healthy" is answered | one table of every pod across every cluster; CrashLoop/OOM visible fleet-wide | +| **Deployments** | the unit of "is my service rolled out everywhere" | "is `payments` at the same image/replicas across all clusters?" in one view | +| **Events** | the "what just broke" firehose | fleet-wide event stream, sortable by cluster/severity/age — cross-cluster triage | +| **Nodes** | capacity & the node-level "why is my pod pending" | node pressure/allocatable across the fleet; find the hot cluster fast | + +**Tier 2 — the rest of the everyday kinds (same slice or fast-follow):** +Services · StatefulSets · DaemonSets · ReplicaSets · Ingresses · ConfigMaps · Secrets *(values +masked by default, reveal is explicit + audited-to-nowhere since local)* · Jobs · CronJobs · +Namespaces · PersistentVolumeClaims · HPAs. + +**Tier 3 — on demand (generic object lens):** +Any other kind via `:` using the generic table renderer (from the API's server-side +print columns), so the tool is never "missing" a resource even before it has a bespoke lens. + +**Tier-1 column specs** (the columns GR reads without thinking; `wide` adds the parenthesized ones): + +- **Pods:** `CLUSTER · NAMESPACE · NAME · READY · STATUS · RESTARTS(+age of last) · AGE · (NODE · IP · CPU · MEM)` +- **Deployments:** `CLUSTER · NAMESPACE · NAME · READY · UP-TO-DATE · AVAILABLE · AGE · (IMAGE · SELECTOR)` +- **Events:** `CLUSTER · NAMESPACE · LAST-SEEN · TYPE · REASON · OBJECT · MESSAGE(truncated)` — default sort: last-seen↓; toggle group-by-cluster. +- **Nodes:** `CLUSTER · NAME · STATUS · ROLES · AGE · VERSION · (CPU%/alloc · MEM%/alloc · PODS/max · TAINTS)` + +Sorting, filtering, and the cluster column are uniform across all lenses (one table component). + +--- + +## 6. Per-pod table stakes (F11.5 / #35) and the 7-verb mapping + +These are the commodity single-cluster ops whose *absence* drove the Lens exodus. They are present +so the local tool is **complete** — deliberately **not** where Sith tries to out-feature Headlamp. +Every one runs as an ordinary K8s API call against the selected context **with the user's own +kubeconfig identity**. None is a governed typed intent; none touches a PEP/PDP; none carries +fleet-action semantics (`SITH-NOTION.md` F11.5, `CONVENTIONS.md` §7.4). + +### 6.0 Mapping onto Architect A's 7-verb source-adapter contract + +Architect A owns the 7-verb source-adapter contract. Its **read triad is `discover` / `read` / +`query`** (given). The remaining verbs are the **streaming / interactive** capabilities. This spec +does **not** define the contract; it maps each per-pod UX action onto a verb and, where a verb's +*exact name* is A's to fix, names the **capability** and flags the binding in §11. + +| UX action | Capability needed from the source adapter | Read triad or streaming? | Binding (this spec's assumption; A confirms the name) | +|---|---|---|---| +| **List/select a pod** | enumerate + fetch object | `discover` (enumerate kinds/objects) + `read` (one object) | read-side — already required for the lens; no new verb | +| **describe** (§6.4) | fetch object + fetch its events | `read` (object) + `query`/`read` (events for object) | read-side, composed; **no streaming verb** | +| **get-yaml view** (§6.5) | fetch raw object | `read` | read-side | +| **get-yaml edit → apply** (§6.5) | write one object back | **not a read verb** | a **write/apply capability** on the adapter, used *locally with the user's identity*; explicitly **outside** the governed closed-vocabulary path. Flagged §11-D1. | +| **logs tail + follow** (§6.1) | open a log byte-stream | **streaming verb** (e.g. `logs`) | streaming — A's verb; UX needs follow + tail + since + container select | +| **exec** (§6.2) | open a bidirectional TTY stream | **streaming verb** (e.g. `exec`) | streaming — A's verb; UX needs interactive `-it` | +| **port-forward** (§6.3) | open a tunneled TCP stream | **streaming verb** (e.g. `port-forward`) | streaming — A's verb; UX needs local:remote mapping + lifecycle | + +**Consequence for A's contract (flagged, not decided here):** the read triad `discover/read/query` +covers list/select/describe/yaml-view. The three long-lived **streaming** capabilities (logs, exec, +port-forward) plus a **local write/apply** for YAML-edit are what F11.5 needs beyond reads. If A's +"7 verbs" are `{discover, read, query, watch, logs, exec, port-forward}`, then: describe & yaml-view +compose from `read`/`query`; the live lens uses `watch`; logs/exec/port-forward map 1:1; and +**YAML-edit's write** is the one capability that has no home in a read-only 7-verb set — §11-D1 +resolves whether it rides a separate local-apply capability or is deferred. **The builder must not +route YAML-edit through any governed intent path** regardless (invariant §7.4). + +### 6.1 Logs (tail + follow) + +- **Entry:** `l` on a selected pod (TUI) · `sith logs --context [-f]` (CLI). +- **Screen:** full-height log pane; header `pod · container · context · ns`; footer keyhints. +- **Controls:** `f` toggle follow (live tail) · `w` wrap · `/` in-log filter (client-side, instant) + · `t` timestamps · `p` previous-container logs (post-crash) · container picker if multi-container + (`c` cycles, or a picker on entry) · `--since`/`--tail` on CLI, `s`/`n` prompts in TUI. +- **Cache-first note:** logs are **not** cached (they're a live byte stream, not fleet facts) — this + is the one place the UI legitimately opens a stream on demand. Opening is fast; the *fleet* views + it returns to are still cache-served. Follow uses the streaming verb; a bounded tail uses a + single ranged read. +- **Multi-pod convenience:** selecting N pods (deployment drill) and pressing `l` opens a **merged, + cluster/pod-prefixed** log stream (`[prod-us/payments-55f8] …`) — a fleet-native affordance k9s + lacks. (Merged-follow is bounded to a sane N; beyond it, prompt.) + +### 6.2 Exec (interactive shell) + +- **Entry:** `s` on a selected pod (TUI) · `sith exec --context -it -- sh` (CLI). +- **Behavior:** allocates a PTY, drops into the container with the user's identity; on exit returns + to the exact fleet view/scope the user left. Container picker for multi-container pods. +- **Boundary (load-bearing):** this is the **user's own kubectl-exec-equivalent**, run with their + kubeconfig identity. It is **never** dispatched as a typed intent, never sees the PEP, and the + closed-vocabulary / no-shell rule that binds *governed* actions does **not** apply to it — because + it isn't a governed action. A code-level assertion (Slice 3 acceptance) proves local exec does not + route through any intent/PEP path. +- **Explicit context required** (§5.2): you exec into a *named* cluster's pod; no implicit "current". + +### 6.3 Port-forward + +- **Entry:** `f` on a selected pod/service (TUI) · `sith port-forward pod/ --context 8080:80`. +- **Behavior:** opens the forward, shows it in a **persistent forwards panel** (`:pf` or a header + badge `⇄2`) listing every active forward with `LOCAL → CLUSTER/ns/target:REMOTE · uptime`; `x` + tears one down. Forwards survive lens/scope changes (they're session-scoped, not view-scoped) and + are all torn down on exit with a confirm if any are live. +- **Fleet affordance:** the forwards panel makes "I have tunnels into three clusters right now" + legible — a thing that is invisible and error-prone in per-cluster tools. + +### 6.4 Describe + +- **Entry:** `d` on any object · `sith describe / --context `. +- **Behavior:** the familiar `kubectl describe` layout (spec highlights + status + **events for this + object**), rendered read-only in a scrollable pane. Composes from the read triad: `read` the + object + `read`/`query` its events (§6.0). No streaming, no write. + +### 6.5 Get-YAML (view / edit) + +- **View:** `y` · `sith yaml / --context ` → raw object YAML, syntax-highlighted, + scrollable, copy-friendly. Pure `read`. Secrets masked unless explicitly revealed. +- **Edit:** `e` (or `y` then `e`) opens the YAML in `$EDITOR`; on save, applies back to the *named + context with the user's identity* — the local `kubectl edit` equivalent. This is the **one** + per-pod action that **writes**, and it is the delicate one: + - It is a **local convenience with the user's own RBAC**, not a governed intent (invariant §7.4). + - Its adapter capability (a local apply/patch) is the item **§11-D1** flags for Architect A: it + has no home in a read-only verb triad. Until D1 resolves, the builder ships **view now, edit + behind the same local-identity apply capability** — and edit **must not** be wired through any + PEP/intent path. + - A dry-run/diff preview before apply (`kubectl diff` equivalent) is shown; destructive/immutable + field edits surface the API's own rejection verbatim (fail-safe, no client-side guessing). + +--- + +## 7. Cross-cluster search & navigation + +The wedge's signature capability (F11.4). One query, whole fleet, coverage-honest. Renders entirely +from the store — search is a filter over cached facts, never a fan-out per keystroke. + +### 7.1 The command bar (`:` and cmd-K / `Ctrl-K`) + +- `:` opens the **command/nav bar** (k9s idiom): `:pods`, `:deploy`, `:nodes`, `:ctx prod-eu`, + `:refresh`, `:pf`, `:help`, `:q`. +- `Ctrl-K` (or `:` then type) opens **fuzzy fleet navigation** — one prompt that fuzzy-matches across + **clusters, namespaces, and object names at once**, so `pay eu` jumps to `payments` in `prod-eu` + without the user first choosing a cluster. This is the "ask once, across everything" affordance. + +### 7.2 Search grammar (fuzzy + structured, cache-served) + +`/` filters the current lens; `Ctrl-K` / `sith search` searches the whole fleet. Both accept plain +fuzzy text **and** structured tokens that compose (AND semantics): + +| Token | Matches | Example | +|---|---|---| +| *(bare text)* | fuzzy over name / namespace / labels | `payments` | +| `ctx:` / `cluster:` | cluster scope (name, glob, or label set) | `ctx:prod-*`, `cluster:@env=prod` | +| `ns:` | namespace | `ns:payments` | +| `status:` | phase/health | `status:CrashLoopBackOff`, `status:!Running` | +| `image:` | container image (substring / repo / tag) | `image:nginx:1.21`, `image:*log4j*` | +| `node:` | scheduled node | `node:n-12` | +| `label:` | label selector | `label:app=payments` | +| `restarts:` | numeric compare | `restarts:>5` | + +`image:` is the seam to the **vuln-fix daily task** (`BUILD-SEQUENCE.md`): "which clusters run image +X" is `sith search image:` today, and deepens into fleet-wide CVE search as F2.4 matures. + +### 7.3 Correlation queries (the differentiator) + +`sith correlate ` / `:correlate` answers fleet-wide questions single-cluster tools structurally +cannot — routed through the F2.3/#10 correlation engine over the cached model: + +- `deploy/payments status!=Healthy` → **every cluster where `payments` is degraded**, one answer. +- `image:*log4j*` → **every cluster/workload running a matching image** (the vuln sweep). +- `pods restarts:>10 in 1h` → fleet-wide flapping-pod roll-up. + +Results are a table with a `CLUSTER` column and a **coverage line** (§4). If a cluster couldn't be +evaluated (stale/unreachable), the answer says `covered 9/11` — it is never presented as a complete +"nowhere is `payments` degraded" when two clusters weren't even reached. + +### 7.4 Navigation from a result + +Any search/correlation row is a live handle: Enter drills into it (jumps scope to that cluster+ns and +selects the object), and all per-pod actions (§6) are available directly from the result — so "find +the crashing `payments` pod across 11 clusters → tail its logs" is: `Ctrl-K payments status:Crash` → +Enter → `l`. Three steps, no manual context switching. + +--- + +## 8. Screens & states (catalog) + +| # | Screen | Purpose | Key states | +|---|---|---|---| +| S1 | **Fleet resource list** (default) | the lens table across the fleet | cold / warming / warm / degraded / offline (§3.3); empty-lens; single-cluster-scoped (cluster col hidden) | +| S2 | **Coverage popover** (`c`) | per-cluster reachability & sync age | all-fresh (dismisses to "all good"); some-stale; some-unreachable-with-error-detail | +| S3 | **Command / nav bar** (`:` / `Ctrl-K`) | jump lens, scope cluster, fuzzy fleet search | empty; typing (live fuzzy preview); no-match; ambiguous (list) | +| S4 | **Search / correlation results** | cross-cluster answer | results+coverage; zero-match (with coverage — "0 matches across 9/11" ≠ "none"); partial-coverage warning | +| S5 | **Object detail / describe** (`d`) | read-only object + events | loaded; object-vanished-during-view (mark stale, offer refresh) | +| S6 | **YAML view/edit** (`y`/`e`) | raw YAML; edit→diff→apply | view; editing; diff-preview; apply-ok; apply-rejected (API error verbatim) | +| S7 | **Logs** (`l`) | tail/follow, single or merged | streaming; follow-on/off; container-switch; stream-ended (pod gone); reconnect-on-drop | +| S8 | **Exec** (`s`) | interactive shell | connecting; live PTY; exited (→ return to prior scope); connect-failed (perm/identity error verbatim) | +| S9 | **Port-forwards panel** (`:pf`) | manage active forwards | none; N-active; a forward dropped (mark, offer re-establish) | +| S10 | **Help / keybindings** (`?`) | discoverability | context-sensitive to current screen | + +**State-design laws (apply to every screen):** +- *Empty ≠ error ≠ unreachable.* "0 pods here" (empty), "couldn't reach this cluster" (unreachable), + and "something broke" (error) are three different messages with three different affordances. +- *Zero-match carries coverage.* A search returning nothing shows `0 matches · covered 9/11` so the + user knows whether "nothing matched" or "we couldn't look everywhere." +- *Object-vanished is normal.* Pods die; a detail/log view whose object disappeared says so and + offers to pop back, never a stack trace. + +--- + +## 9. Keybindings (k9s-informed, fleet-extended) + +Global (any screen unless a modal owns input): + +| Key | Action | Key | Action | +|---|---|---|---| +| `:` | command / nav bar | `/` | filter current lens | +| `Ctrl-K` | fuzzy fleet search | `Esc` | back / clear filter / close modal | +| `?` | help (context-sensitive) | `q` / `Ctrl-C` | quit (confirm if forwards live) | +| `0` | scope: all clusters | `1`–`9` | scope: Nth cluster | +| `<` / `>` | cycle cluster-set scopes | `Tab` | next region focus | +| `c` | coverage popover | `Ctrl-R` | refresh current scope (hint) | +| `Space` | select/deselect row | `Ctrl-A` | select all in view | +| `g` / `G` | top / bottom | `↑`/`↓`/`PgUp`/`PgDn` | move / page | +| `Enter` | drill into row | `s`ort via header `[` `]` | cycle / reverse sort | +| `:pause` | freeze store (stable read) | `:resume` | resume watches | + +Resource / object actions (on a selected row): + +| Key | Action | Maps to (§6) | +|---|---|---| +| `l` | logs (tail/follow; merged if multi-select) | §6.1 · streaming `logs` | +| `s` | exec / shell | §6.2 · streaming `exec` | +| `f` | port-forward | §6.3 · streaming `port-forward` | +| `d` | describe | §6.4 · `read`+`query` | +| `y` | YAML view | §6.5 · `read` | +| `e` | YAML edit → diff → apply | §6.5 · local write (§11-D1) | +| `Enter` | drill (pod→containers, deploy→pods, node→pods) | `discover`/`read` | + +Logs pane: `f` follow · `w` wrap · `t` timestamps · `p` previous-container · `c` container · `/` in-log filter. +Conventions match k9s where they exist so muscle memory transfers; new keys (`0`–`9`, `<`/`>`, `c`) +are all fleet-scoping additions, chosen not to collide with k9s reflexes. + +--- + +## 10. Acceptance criteria + +### 10.1 F11.2 — cache-first fleet render (CLI + TUI) — Slice 2 + +- **AC-2.1 (single render source).** Every view, sort, filter, lens-change, and navigation renders + from the store; a test proves **no Kubernetes API call is issued on the interaction path** (only + the background sync loop calls the API). +- **AC-2.2 (latency).** Warm-store interactions render in **< 100 ms** p95; **first paint < 250 ms** + regardless of context count or slowest-context latency (non-blocking fan-out). +- **AC-2.3 (no spinner-first).** Cold/warming states show already-hydrated clusters immediately with + a per-cluster warming indicator; the screen is never a single blocking spinner. +- **AC-2.4 (CLI/TUI parity).** `sith get --all-clusters` returns the **same aggregated + answer** the TUI lens shows for the same scope (golden-parity test). +- **AC-2.5 (coverage honesty).** Every aggregate view and every search/correlation result renders a + coverage line; a run with ≥1 unreachable context shows `covered N/M · … unreachable()` and + **never** presents the result as complete (reuses `Coverage` + F2.5). +- **AC-2.6 (staleness surfaced).** Stale clusters render last-known values **with** a staleness + marker (never blanked); crossing a threshold updates the marker without a full redraw. +- **AC-2.7 (offline).** With no network, the TUI still renders last-known fleet + an offline banner; + it does not error out or clear the screen. +- **AC-2.8 (cross-cluster search/correlation).** One query returns a **correct** answer over **≥ 2 + contexts**; coverage gaps are flagged in the result (F11.4). +- **AC-2.9 (lens order).** Tier-1 lenses (pods, deployments, events, nodes) are present and are the + default/first-reachable lenses; the generic lens renders any other kind via server print columns. + +### 10.2 F11.5 — per-pod table stakes — Slice 3 + +- **AC-3.1 (the four work).** logs (stream + tail + follow), exec (interactive), port-forward, and + YAML view/edit each work per context using the **user's own kubeconfig identity**. +- **AC-3.2 (describe + yaml).** describe (object + its events) and get-yaml (raw view) render + read-only; both compose from the read triad (`read`/`query`) — no streaming verb, no write. +- **AC-3.3 (verb mapping).** Each action maps onto Architect A's contract per §6.0: describe/yaml-view + = read-side; logs/exec/port-forward = streaming verbs; the live lens = `watch`. The mapping is + recorded and any unresolved binding is listed in §11. +- **AC-3.4 (local ≠ governed — code-level).** A test asserts **no** per-pod action routes through any + intent / PEP / closed-vocabulary path; local exec/edit are the user's own kubectl-equivalents + (invariant `CONVENTIONS.md` §7.4). +- **AC-3.5 (explicit context for actions).** exec / port-forward / yaml-edit require an explicit + `--context`; there is no implicit "current cluster" for an action. +- **AC-3.6 (YAML-edit safety).** edit shows a diff/dry-run before apply; API rejections surface + verbatim (fail-safe, no client-side guessing); edit does not touch any governed path. +- **AC-3.7 (return-to-scope).** After exec/logs/detail, focus returns to the exact fleet scope/row + the user left (the fleet is the home; the pod action is a visit). + +### 10.3 Trust invariants (verified continuously; proven in Slice 5) + +- **AC-T.1** No screen or action opens egress except to the user's own clusters (network-egress test, + Slice 5). **AC-T.2** loopback-only holds (relevant when §12's `sith ui` lands). **AC-T.3** no + action copies a kubeconfig/credential off the machine. **AC-T.4** no account, no telemetry. + +--- + +## 11. Open decisions (recorded for the PR / owner) + +- **D1 — YAML-edit's write capability vs. a read-only 7-verb contract (for Architect A).** + describe/yaml-view/list/select all compose from `discover/read/query`. logs/exec/port-forward are + streaming verbs. **YAML-edit is the one F11.5 action that writes.** If A's 7 verbs are read+stream + only, edit needs either (a) a distinct **local-apply capability** on the source adapter (used with + the user's identity, explicitly outside the governed vocabulary), or (b) deferral of *edit* (ship + *view* in Slice 3, edit as a fast-follow). **Recommendation:** (a) — a named local-apply capability + gated to local mode + user identity, never wired to a PEP. Decision belongs to A + owner. +- **D2 — exact streaming verb names.** This spec assumes `{logs, exec, port-forward}` and a `watch` + for the live lens. If A names them differently, the §6/§9 mapping updates mechanically; no UX + change. (Flagged so the builder waits on A's names before wiring, not before designing.) +- **D3 — Q12 hero surface.** `BUILD-SEQUENCE.md` locks **TUI-first**, `sith ui` fast-follow (Slice 4). + This spec is written TUI-first with a CLI peer. If the owner overrides to web-first, §5.3/§8 port + to the web surface unchanged (same store, same states); only the render target moves. +- **D4 — default cross-cluster scope for the TUI.** This spec defaults the **TUI** to all-clusters + (the wedge's point) while the **CLI** requires explicit `--all-clusters` (script safety). Confirm + the asymmetry is desired; alternative is symmetric explicit-opt-in (safer, less wow). +- **D5 — merged multi-pod logs bound.** Merged-follow across selected pods needs an N cap for + readability/stream limits. Proposed default: warn/prompt beyond 20 streams. Owner may tune. +- **D6 — staleness thresholds.** Defaults (`fresh ≤15 s`, `stale ≤2 m`, `very-stale >2 m`) are + first-guess; validate against real multi-cloud sync cadences once Slice 1 measures them. +- **D7 — secret reveal in local mode.** Secrets are masked by default; reveal is explicit. Since + local mode has no audit sink, reveal is a pure local action — confirm no reveal-logging is wanted + (consistent with the no-telemetry promise). + +--- + +## 12. Boundaries & the web surface (F11.3, not this spec) + +- **Not in this spec:** `sith ui` (Slice 4). But the web surface is **"the same engine, a second + face"** — it must render the **same store**, the **same lens order** (§5.5), the **same coverage/ + staleness semantics** (§§3–4), and the **same per-pod actions** (§6) with the **same local-identity, + non-governed** boundary. Anything the web UI shows that the TUI/CLI can't is a divergence bug. +- **Not in local mode at all:** governed fleet actions, typed intents, approvals, the PEP/PDP, any + hub concept. Local mode is read + the user's own per-pod conveniences. The *same person* acting + through governance is a **hub-mode** experience (E4/E5), out of Phase L. +- **Deliberately not chased:** out-feature-ing Headlamp on single-cluster depth. The per-pod set is + the *floor* that makes the tool complete; the fleet view is the *ceiling* that makes it worth + switching. This spec spends its complexity budget on the fleet, per Principle 2. + +--- + +*End — F11.2 + F11.5 Local Fleet UX. Implements against Slice 0's `fleet.Source`/`FleetResult`; +consumes Slice 1's local-kubeconfig adapter; delivered as Slice 2 (render/search) + Slice 3 +(per-pod). References Architect A's 7-verb source-adapter contract (§6.0); open bindings in §11.* diff --git a/docs/specs/F2.1-source-adapter-contract.md b/docs/specs/F2.1-source-adapter-contract.md new file mode 100644 index 0000000..f29fbad --- /dev/null +++ b/docs/specs/F2.1-source-adapter-contract.md @@ -0,0 +1,840 @@ +# Spec — F2.1 Source-Abstract Fleet Model + Connector / Source-Adapter Contract + +**Status:** design locked, ready for build · **Date:** 2026-07-10 +**Author role:** Architect (design/spec only — a builder session implements the Go) +**Issue:** F2.1 (#38) · **Epic:** E2 read federation (#20) · **Also feeds:** E11 (#29), E12 (#30) +**Branch:** `docs/f2.1-source-adapter-contract` off `dev` · **PR target:** `dev` +**Prereqs to read:** +[`../SITH-NOTION.md`](../SITH-NOTION.md) E2 / F2.1–F2.5, E11, E12 · +[`SLICE-0-foundation.md`](SLICE-0-foundation.md) §3.4 (the `fleet.Source` seam) · +[`../adr/0004-typed-intent-action-model.md`](../adr/0004-typed-intent-action-model.md) · +[`../adr/0005-ai-mcp-ardur-pdp.md`](../adr/0005-ai-mcp-ardur-pdp.md) · +[`../CONVENTIONS.md`](../CONVENTIONS.md) · +[`../BUILD-SEQUENCE.md`](../BUILD-SEQUENCE.md) Slice 1. + +> **Scope of this document.** This is the *contract* — the Go interfaces, the capability model, +> the normalized fleet/resource model, the operational-graph shape, and the first adapter's +> behavior. It **specifies**, it does not implement. It sits **above** the Slice-0 walking +> skeleton: it references and extends the `fleet.Source` seam Slice 0 ships (§3.4 of +> [`SLICE-0-foundation.md`](SLICE-0-foundation.md)); it does **not** redefine the foundation. +> Product identity is **ArdurAI** — never any predecessor name. + +--- + +## 0. Why this is the most important seam in Sith + +Every integration Sith will ever grow — Kubernetes, GitHub, ArgoCD, Prometheus, Elasticsearch, +AWS, Helm, Kustomize, Grafana, OTel, OpenShift, Azure, GCP, OpenSearch, Splunk, Fluentd, Istio, +Docker — plugs into **one shape**. If that shape is right, adding a connector is *additive*: a new +package that implements the verbs it supports and registers itself. Nothing above the seam +changes. If the shape is wrong, every new integration is architectural rework, and Sith becomes +the 12-provider pass-through sprawl the reshape explicitly rejects (`SITH-NOTION.md` §"what we +kept / cut"). + +The corpus this design reuses — built from real fleet-ops incidents — converged on a **7-verb +connector contract**: + +``` +discover · read · query · diff · plan · execute · verify +``` + +plus a **Capability** enum and typed **ResourceRef / Evidence / ActionPlan / Verification** +values. This document translates those into idiomatic Go and grounds the first implementation — +the **local-kubeconfig adapter** — in Sith's day-0 local mode (E11 / Slice 1). + +**The load-bearing invariant:** *everything above the source is one code path.* The local client +(E11), the hub (E1–E10), the MCP read tools (E7/F7.1), and the cost overlay (E13) all consume the +same normalized model over the same contract. A fact's origin — a local kubeconfig context or an +OCM-brokered spoke — is a `SourceKind` field, not a fork in the code. + +--- + +## 1. Package layout & the one-way dependency rule + +Three packages, one dependency direction (no cycles — this is a hard rule): + +``` +internal/fleet/ # the DATA model: ResourceRef, Evidence, Fact, Query, Graph, + # FleetResult, Coverage, Cluster (+ the Slice-0 Source seam) + # depends on: stdlib only +internal/connector/ # the CONTRACT: Connector, the verb interfaces, Capability, + # Registry, Intent, ActionPlan, Verification, the AsSource bridge + # depends on: internal/fleet (one way, never the reverse) +internal/connector/kubeconfig/ # the FIRST adapter (Slice 1) + # depends on: internal/connector, internal/fleet, client-go +``` + +**Dependency direction: `kubeconfig → connector → fleet`, never back.** `fleet` is pure data and +knows nothing about connectors; `connector` references `fleet` value types (`ResourceRef`, +`Evidence`, …); adapters reference both. This is why the `fleet.Source ⇄ connector.Reader` bridge +(§7) lives in `connector`, not in `fleet` — putting it in `fleet` would create the cycle. + +> **Relationship to Slice 0 (do not redefine the foundation).** Slice 0 already ships +> `internal/fleet/{model.go,source.go,stub.go}` with `Source`, `FleetResult`, `Cluster`, +> `Coverage`. This spec **adds** types to `internal/fleet` (`ResourceRef`, `Evidence`, `Fact`, +> `Query`, `Graph`, …) and **introduces** `internal/connector`. It does not change the existing +> `Source` interface signature. `Source` remains the narrow *snapshot* seam the CLI depends on; +> the connector contract is the full seam beneath it, and `Source` is derived from it (§7). + +--- + +## 2. The normalized fleet / resource model (`internal/fleet`) + +The model is **source-abstract**: nothing in it names Kubernetes, ArgoCD, or Prometheus. Adapters +map their native shapes *into* this model; consumers only ever see this model. + +### 2.1 Addressing — `ResourceRef` + +One source-abstract address for any resource anywhere in the fleet. + +```go +package fleet + +// ResourceRef is a source-abstract address for one resource in the fleet. +// It is stable enough to round-trip (Read(ref) after a Query returns ref) and +// opaque enough that no consumer must understand a source's native identifiers. +type ResourceRef struct { + SourceKind string `json:"source_kind"` // connector kind that owns this address space, e.g. "local-kubeconfig" + Scope string `json:"scope"` // cluster/context/spoke within the source (a kubeconfig context name) + Kind string `json:"kind"` // normalized resource kind: "Deployment","Pod","Rollout","Alert","Image",… + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + // Attributes carries adapter-specific address bits a consumer must not + // interpret but an adapter needs to resolve the ref (GVR, UID, tool object id, + // Prometheus series selector, ARN). Never load-bearing for equality above the seam. + Attributes map[string]string `json:"attributes,omitempty"` +} + +// Equal compares the source-abstract identity (SourceKind/Scope/Kind/Namespace/Name), +// ignoring Attributes so two adapters/versions addressing the same resource collide. +func (r ResourceRef) Equal(o ResourceRef) bool +func (r ResourceRef) String() string // "source:scope/kind/namespace/name" for logs & audit +``` + +### 2.2 The fact taxonomy — `FactKind` + +Closed set (extending it is a deliberate, reviewed change — mirrors the closed-vocabulary +discipline of ADR-0004). These are exactly the kinds E2 normalizes into (`SITH-NOTION.md` F2.1 +step 3) plus `cost` (E13). + +```go +type FactKind string + +const ( + FactInventory FactKind = "inventory" // workload objects and their spec/status + FactHealth FactKind = "health" // derived condition: Healthy/Degraded/Progressing/Unknown + FactAlert FactKind = "alert" // firing alerts / anomaly signals + FactDrift FactKind = "drift" // desired-vs-observed divergence (GitOps) + FactCVE FactKind = "cve" // image/CVE findings + FactCost FactKind = "cost" // per-cluster cost read-overlay (E13) +) + +func (k FactKind) Valid() bool // fail-safe: unknown kind => false, never defaulted +``` + +### 2.3 Observed state with provenance — `Evidence` and `Fact` + +`Evidence` is the unit every `Read` returns: observed state **plus** the provenance that makes a +partial or stale view impossible to mistake for a complete, fresh one (`SITH-NOTION.md` F2.2 / +F2.5). `Fact` is `Evidence` after it has been stamped into the fleet-model cache. + +```go +// Evidence is observed state plus provenance. It is what a source hands back; +// honesty about freshness and source is structural, not optional. +type Evidence struct { + Ref ResourceRef `json:"ref"` + Kind FactKind `json:"kind"` + Observed json.RawMessage `json:"observed"` // normalized payload for Kind (typed per FactKind by consumers) + ObservedAt time.Time `json:"observed_at"` // when the source last saw this (zero => never observed) + Source string `json:"source"` // the scope (cluster/context) that produced it + Provenance Provenance `json:"provenance"` // how to trace/deep-link back to the raw read +} + +// Provenance lets a brokered read-through deep-link to the tool's own UI instead +// of re-skinning it (E12/F12.2), and lets audit trace any fact to its origin. +type Provenance struct { + Adapter string `json:"adapter"` // Connector.Kind() + ProtocolV string `json:"protocol_version"` // connector protocol version that produced it + NativeID string `json:"native_id,omitempty"` // the source's own id (UID, series, object id) + DeepLink string `json:"deep_link,omitempty"` // URL into the tool's own UI (brokered read-through) + Collector string `json:"collector,omitempty"` // read-session / worker id, for debugging +} + +// Fact is Evidence after the fleet-model store stamps it (F2.2): workspace-scoped, +// freshness-derived. In local mode Workspace is the implicit single-user workspace. +type Fact struct { + Evidence + Workspace string `json:"workspace"` // tenancy anchor (E1); implicit local ws in local mode + Stale bool `json:"stale"` // derived from ObservedAt vs the freshness threshold (F2.5) + StaleFor string `json:"stale_for,omitempty"` // human age past threshold, e.g. ">10m" +} +``` + +> **Why `json.RawMessage` for `Observed`.** The model must admit heterogeneous payloads +> (a Deployment status, an alert, a CVE finding) without the `fleet` package importing every +> adapter's schema. Consumers decode by `FactKind` into typed views. This keeps `fleet` +> dependency-free and additive: a new `FactKind` payload does not touch the core types. + +### 2.4 Selection — `Query` and `QueryResult` + +`Query` expresses cross-cluster selection *without naming a single resource* — the mechanism +behind F2.3/F11.4 ("every cluster where `payments` is Degraded", "which contexts run image X"). + +```go +type Query struct { + Kinds []FactKind `json:"kinds,omitempty"` // restrict to fact kinds + Scopes []string `json:"scopes,omitempty"` // restrict to specific clusters/contexts; empty => all reachable + Selector Selector `json:"selector,omitempty"` // structured predicate (NOT a free-form string — fail-safe) + Limit int `json:"limit,omitempty"` +} + +// Selector is a small, typed predicate set — deliberately not an arbitrary expression +// language (unschema'd input is refused, CONVENTIONS §7.5). Adapters translate it to their +// native filter; the core validates it. +type Selector struct { + ResourceKind string `json:"resource_kind,omitempty"` // "Deployment","Pod",… + Namespace string `json:"namespace,omitempty"` + NamePrefix string `json:"name_prefix,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Health string `json:"health,omitempty"` // "Degraded","Healthy",… + Image string `json:"image,omitempty"` // for cve/inventory image search + CVE string `json:"cve,omitempty"` +} + +type QueryResult struct { + Facts []Fact `json:"facts"` + Coverage Coverage `json:"coverage"` // which scopes answered, which were stale/unreachable +} +``` + +### 2.5 Coverage & the fleet snapshot (Slice-0 types, unchanged) + +`Coverage`, `Cluster`, and `FleetResult` are the Slice-0 types (`SLICE-0-foundation.md` §3.4). +This spec keeps them verbatim and only *extends* `Coverage` additively so a partial view is never +mistaken for a complete one — the single most dangerous failure of a federated read model +(F2.5 guardrail). + +```go +// Coverage summarizes reachability + freshness across the queried scopes. +// (Slice-0 fields Requested/Reachable/Unreachable retained; Stale added, additive.) +type Coverage struct { + Requested int `json:"requested"` + Reachable int `json:"reachable"` + Unreachable []string `json:"unreachable,omitempty"` // scopes that failed to answer + Stale []string `json:"stale,omitempty"` // scopes that answered but past the freshness threshold +} + +// Complete reports whether every requested scope answered fresh — the honesty check +// callers (and the abstention path, E5) gate on. +func (c Coverage) Complete() bool // Requested==Reachable && len(Stale)==0 +``` + +`FleetResult` (Slice-0) stays as-is: `{ Clusters []Cluster; Coverage Coverage }`. It is the +*snapshot* view; §7 shows how a connector produces it. + +### 2.6 The operational graph — `Graph` + +Correlation (F2.3) and "explain this incident across clusters" are graph questions, not row +scans. The model therefore carries an **operational graph**: nodes are resources (each stamped +with its latest `Evidence`), edges are **typed relations** between them. The graph is *derived* +from facts by the fleet model, not stored by adapters — adapters emit facts and, where they know +one, relation hints; the model assembles the graph. + +```go +type Graph struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` +} + +type Node struct { + Ref ResourceRef `json:"ref"` + Fact Fact `json:"fact"` // latest observed state for the node +} + +// Relation is a closed set of typed edges — additive, but every value is meaningful +// across sources (source-abstract). No free-form edge labels. +type Relation string + +const ( + RelOwns Relation = "owns" // Deployment -> ReplicaSet -> Pod + RelRoutesTo Relation = "routes_to" // Service/Ingress -> Pod + RelBackedBy Relation = "backed_by" // Pod -> Node / PVC + RelDeployedFrom Relation = "deployed_from" // workload -> Git source / ArgoCD app (drift) + RelRunsImage Relation = "runs_image" // Pod -> Image (CVE search join) + RelAlertsOn Relation = "alerts_on" // Alert -> target resource + RelCostsFor Relation = "costs_for" // cost fact -> workload/namespace (E13) +) + +type Edge struct { + From ResourceRef `json:"from"` + To ResourceRef `json:"to"` + Rel Relation `json:"rel"` +} +``` + +> **Why a graph and not just rows.** "Which clusters run image X with CVE Y" is a `RunsImage` +> join; "why is `payments` Degraded everywhere" walks `Owns`/`RoutesTo`/`AlertsOn` from the +> workload. A graph makes correlation (E2's differentiator vs. single-cluster consoles) a +> first-class traversal instead of an ad-hoc query per question. **Slice 1 does not need to build +> the graph** — it emits facts and the obvious ownership relation hints; the graph assembler is +> Slice 2 (F11.4 / F2.3). The types are specified here so the seam is fixed and Slice 2 is +> additive. + +--- + +## 3. The connector contract — the 7 verbs as Go (`internal/connector`) + +The corpus's 7-verb contract translated to idiomatic Go. The key idiomatic move: **the 7 verbs +are not one fat interface.** They are grouped into small, composable, capability-scoped +interfaces, so a read-only source (Prometheus, kubeconfig) implements only what it can do, and the +compiler — not a runtime check alone — proves what a connector supports. + +### 3.1 The base `Connector` + +```go +package connector + +// Connector is the identity every source-adapter shares. The verbs live in the +// capability interfaces below; a bare Connector does nothing but identify itself. +type Connector interface { + // Kind is the stable, unique adapter identifier — one canonical per target + // tool (E12/F12.3), e.g. "local-kubeconfig", "ocm-spoke", "prometheus", "argocd". + Kind() string + + // Capabilities declares which verbs this connector implements. The registry + // cross-checks each declared capability against the matching interface at + // registration: a declared capability whose interface is unsatisfied is a + // registration error (fail-safe — you cannot advertise a verb you can't run). + Capabilities() []Capability + + // Descriptor is static metadata for the registry, taxonomy enforcement, and audit. + Descriptor() Descriptor +} + +// Descriptor pins a connector into E12's closed taxonomy and versioning discipline. +type Descriptor struct { + Kind string `json:"kind"` + ConnKind ConnectorKind `json:"connector_kind"` // read-adapter | brokered-read-through | typed-action (F12.2) + ProtocolV string `json:"protocol_version"` // semver; minor is additive, major is a reviewed break (F12.3) + Owner string `json:"owner"` // named owner — one canonical connector per tool + Capabilities []Capability `json:"capabilities"` + Verbs []string `json:"verbs,omitempty"` // closed-vocab verbs this adapter hosts (typed-action only) +} + +// ConnectorKind is E12/F12.2's closed taxonomy — nothing outside these three exists. +type ConnectorKind string + +const ( + KindReadAdapter ConnectorKind = "read-adapter" // pulls normalized facts into the fleet model + KindBrokeredRead ConnectorKind = "brokered-read-through" // deep-links to the tool's own UI; never re-skins + KindTypedAction ConnectorKind = "typed-action" // maps a closed verb to the tool's API +) +``` + +### 3.2 The read verbs — `Reader` (discover · read · query) + +Every source implements at least `Reader`. These are the day-0 verbs, and the only verbs the +local client ever calls. + +```go +// Reader is the read half of the contract: the three read-only verbs. It is the +// minimum a source implements and the only surface local mode (E11) uses. +type Reader interface { + Connector + + // Discover enumerates the addressable scopes (clusters/contexts/spokes) this + // source exposes and, per scope, the resource kinds it can serve. It is cheap, + // non-mutating, and must be non-blocking per scope: one unreachable scope is + // surfaced in Discovery.Unreachable, never fatal (F11.1 guardrail). + Discover(ctx context.Context) (Discovery, error) + + // Read fetches the current observed state of one addressed resource as Evidence. + // ref must be one a prior Discover/Query surfaced from this source. + Read(ctx context.Context, ref fleet.ResourceRef) (fleet.Evidence, error) + + // Query selects normalized facts across the source's scopes without naming a + // single resource. It fans out across scopes concurrently and returns partial + // results with Coverage rather than failing when a scope is unreachable/stale. + Query(ctx context.Context, q fleet.Query) (fleet.QueryResult, error) +} + +// Discovery is what Discover returns: the shape of the source right now. +type Discovery struct { + Scopes []Scope `json:"scopes"` // reachable scopes and their servable kinds + Unreachable []string `json:"unreachable,omitempty"` // scope names that failed probe (surfaced, not fatal) +} + +type Scope struct { + Name string `json:"name"` // kubeconfig context name in local mode + Kinds []string `json:"kinds"` // normalized resource kinds servable here + Reachable bool `json:"reachable"` + ObservedAt time.Time `json:"observed_at,omitempty"` // when reachability was last confirmed +} +``` + +### 3.3 The write verbs — `Differ · Planner · Executor · Verifier` + +`diff · plan · execute · verify` are the governed write lifecycle. They map directly onto +ADR-0004's discipline (**dry-run first, surface the plan/diff, separate explicit execute, +then confirm**) and the audit ledger (`proposed → approved → dry-run → executed`). **They are not +part of Slice 1** and are **never** reachable from local mode — they are specified here so the +seam is fixed once and the typed-action connectors of E4/E12 slot in additively. + +```go +// Differ computes desired-vs-observed for an addressed resource or set (the "diff" +// that seeds drift facts and precedes any write). Read-only; safe to call anytime. +type Differ interface { + Connector + Diff(ctx context.Context, req DiffRequest) (fleet.Diff, error) +} + +// Planner turns a typed Intent (closed vocabulary — ADR-0004) into a concrete, +// inspectable ActionPlan WITHOUT mutating anything. This is the mandatory dry-run. +type Planner interface { + Connector + Plan(ctx context.Context, intent Intent) (ActionPlan, error) +} + +// Executor applies a previously-produced ActionPlan. CRITICAL: Execute is invoked +// ONLY by the hub PEP after the PDP (Ardur) has approved the intent (ADR-0004 rule 5, +// ADR-0005). It is never called on the local read path; local per-pod ops (F11.5) +// use the user's own kubeconfig identity and are NOT connector executions. +type Executor interface { + Connector + Execute(ctx context.Context, plan ActionPlan) (ExecutionResult, error) +} + +// Verifier checks the post-conditions of an execution (or any asserted state) — the +// "did it actually take effect" confirmation that closes the ledger entry. +type Verifier interface { + Connector + Verify(ctx context.Context, req VerifyRequest) (Verification, error) +} +``` + +Supporting types (the corpus dataclasses, as Go): + +```go +// Intent is the typed, closed-vocabulary write request (ADR-0004). The connector +// never invents an intent; it receives one already validated and signed by the hub. +type Intent struct { + ID string `json:"id"` + Workspace string `json:"workspace"` + Actor string `json:"actor"` + Verb string `json:"verb"` // MUST be in the closed vocabulary (argocd.sync, rollout.promote, …) + Target fleet.ResourceRef `json:"target"` // resolved only within the actor's workspace (ADR-0003) + Args json.RawMessage `json:"args"` // typed + JSON-schema-validated per verb + Justification string `json:"justification"` + EvidenceRefs []fleet.ResourceRef `json:"evidence_refs,omitempty"` + Signature string `json:"signature"` // hub signature; the spoke re-verifies independently +} + +// ActionPlan is the dry-run product of Plan: exactly what Execute will do, made +// inspectable before anyone approves it. Reversible-by-design where possible. +type ActionPlan struct { + IntentID string `json:"intent_id"` + Verb string `json:"verb"` + Target fleet.ResourceRef `json:"target"` + Diff fleet.Diff `json:"diff"` // the change this plan would make + Steps []PlanStep `json:"steps"` // ordered, typed API calls (NEVER shell strings — ADR-0004 rule 2) + Reversible bool `json:"reversible"` + Warnings []string `json:"warnings,omitempty"` +} + +type PlanStep struct { + Description string `json:"description"` + API string `json:"api"` // the typed API call, e.g. "argocd.Application.Sync" + Params json.RawMessage `json:"params"` +} + +type ExecutionResult struct { + IntentID string `json:"intent_id"` + Applied bool `json:"applied"` + StepsDone int `json:"steps_done"` + Observed fleet.Evidence `json:"observed"` // post-execution observed state + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Err string `json:"err,omitempty"` +} + +type VerifyRequest struct { + IntentID string `json:"intent_id"` + Target fleet.ResourceRef `json:"target"` + Expect fleet.Selector `json:"expect"` // the typed post-condition to confirm +} + +// Verification is the corpus's verification dataclass: the post-condition verdict. +type Verification struct { + Satisfied bool `json:"satisfied"` + Observed fleet.Evidence `json:"observed"` + Detail string `json:"detail,omitempty"` + CheckedAt time.Time `json:"checked_at"` +} +``` + +`DiffRequest` / `fleet.Diff`: + +```go +// DiffRequest asks for desired-vs-observed on a target. Desired may come from Git +// (GitOps drift), a manifest, or an Intent's projected end-state. +type DiffRequest struct { + Target fleet.ResourceRef `json:"target"` + Desired json.RawMessage `json:"desired,omitempty"` // omit => source supplies desired (e.g. ArgoCD app spec) +} + +// fleet.Diff (declared in internal/fleet): a structured, source-abstract diff. +type Diff struct { + Ref ResourceRef `json:"ref"` + Drifted bool `json:"drifted"` + Hunks []DiffHunk `json:"hunks,omitempty"` // structured field-level changes, never a raw text blob +} +type DiffHunk struct { + Path string `json:"path"` // JSONPath-ish field path + Observed string `json:"observed"` + Desired string `json:"desired"` +} +``` + +### 3.4 Verb → capability → interface map (the whole contract on one page) + +| Verb | Capability | Interface | Slice 1? | Governed? | +|---|---|---|---|---| +| discover | `CapDiscover` | `Reader` | ✅ yes | read (ungoverned) | +| read | `CapRead` | `Reader` | ✅ yes | read (ungoverned) | +| query | `CapQuery` | `Reader` | ✅ yes | read (ungoverned) | +| diff | `CapDiff` | `Differ` | later | read (ungoverned; seeds drift) | +| plan | `CapPlan` | `Planner` | later | governed dry-run (ADR-0004) | +| execute | `CapExecute` | `Executor` | later | **governed** — PEP-only, post-PDP (ADR-0004/5) | +| verify | `CapVerify` | `Verifier` | later | governed post-condition | + +--- + +## 4. The Capability enum + registry (`internal/connector`) + +### 4.1 Capability + +```go +// Capability names one verb of the contract. A connector both DECLARES a capability +// (Capabilities()) and IMPLEMENTS its interface; the registry enforces the pair. +type Capability string + +const ( + CapDiscover Capability = "discover" + CapRead Capability = "read" + CapQuery Capability = "query" + CapDiff Capability = "diff" + CapPlan Capability = "plan" + CapExecute Capability = "execute" + CapVerify Capability = "verify" +) + +func (c Capability) Valid() bool // fail-safe: unknown capability => false + +// iface returns the interface a capability requires; used by the registry to +// verify declaration matches implementation (see Register). +``` + +### 4.2 Registry — capability-based lookup with fail-safe registration + +```go +// Factory constructs a configured connector. Registration is by factory so the +// registry can (re)build connectors and so a construction error fails registration, +// not first use. +type Factory func() (Connector, error) + +// Registry holds the process's connectors. It enforces E12/F12.3: one canonical +// connector per Kind, versioned, capability-checked. +type Registry struct { /* unexported: map[string]entry + mutex */ } + +func NewRegistry() *Registry + +// Register builds the connector, then FAIL-SAFE checks: +// 1. Descriptor.Kind is non-empty and not already registered (one canonical per tool). +// 2. Descriptor.ConnKind is one of the three (F12.2) — else refused. +// 3. Every declared Capability maps to an interface the concrete type satisfies +// (e.g. CapExecute => value asserts to Executor). A declared-but-unimplemented +// capability is a registration error. A capability implemented but not declared +// is ignored (declaration is the source of truth; you opt in explicitly). +// 4. typed-action connectors declare their closed-vocab Verbs; read/brokered do not. +// Any failure returns an error and registers nothing. +func (r *Registry) Register(f Factory) error + +// ByKind returns the canonical connector for a kind. +func (r *Registry) ByKind(kind string) (Connector, bool) + +// WithCapability returns every connector that BOTH declares AND (re-asserted here, +// defense-in-depth) implements the capability. This is the capability-based lookup +// the read federation and the PEP use to find who can serve a verb. +func (r *Registry) WithCapability(c Capability) []Connector + +// Descriptors lists all registered connectors for `sith connectors` / audit. +func (r *Registry) Descriptors() []Descriptor +``` + +Typed accessors do the interface assertion once, so callers never hand-assert (and never +accidentally skip the capability check): + +```go +// ReaderFor returns the Reader for kind, or an error if kind is unregistered or does +// not implement Reader. Never returns a non-nil Reader that lacks the capability. +func (r *Registry) ReaderFor(kind string) (Reader, error) + +// ExecutorFor is the governed-path accessor: it returns an Executor ONLY if the +// connector declared CapExecute AND implements Executor AND is a typed-action kind. +// The PEP uses this; a read adapter can never be coerced into an executor. +func (r *Registry) ExecutorFor(kind string) (Executor, error) + +// (PlannerFor, DifferFor, VerifierFor analogous.) +``` + +> **Why declaration *and* implementation.** Declaration alone (a slice of strings) could lie; +> implementation alone (type assertion) could silently expose a verb the author never intended to +> ship. Requiring both — checked at registration, re-checked at lookup — means a connector runs a +> verb only when the author explicitly opted in *and* the code exists. This is the same +> fail-safe-allowlist posture ADR-0004 mandates for the write vocabulary, applied to the connector +> surface. + +--- + +## 5. The first adapter — `local-kubeconfig` (`internal/connector/kubeconfig`) + +This is **Slice 1** (F2.1 #38 + F11.1 #32). It is a **read adapter** implementing **only +`Reader`** — the three read-only verbs. No `Differ`/`Planner`/`Executor`/`Verifier`: local mode +has no governed write path (ADR-0004; F11.5 per-pod ops are the user's own kubectl-equivalent, not +connector executions). + +### 5.1 Descriptor + +```go +Descriptor{ + Kind: "local-kubeconfig", + ConnKind: KindReadAdapter, + ProtocolV: "1.0.0", + Owner: "sith-core", + Capabilities: []Capability{CapDiscover, CapRead, CapQuery}, + // no Verbs — read adapter +} +``` + +### 5.2 Behavior + +**`Discover(ctx)` — list contexts (this is F11.1's fan-out entrypoint):** + +1. Resolve kubeconfig sources: honor `$KUBECONFIG` (OS-path-list, colon-joined on Unix) if set, + else `~/.kube/config`. Merge exactly as client-go's loading rules do — do not re-implement + precedence. +2. Enumerate **every context**. Each context → a `Scope{Name: contextName}`. +3. Probe reachability **per context, concurrently, non-blocking**, with a short per-context + timeout (default 2s, configurable). Reachability = a cheap authenticated call + (`/version` or a `SelfSubjectAccessReview`/discovery ping). Honor the context's + **exec-credential plugin locally** (aws/gcloud/az helpers run as kubectl would) — credentials + are read and used **in place; never copied off the machine** (invariant 3, verified by the + Slice-5 egress test). +4. A context whose probe fails or times out → **`Discovery.Unreachable`**, never an error return. + One dead or auth-prompting context must not stall discovery of the others (F11.1 guardrail). +5. Reachable contexts report their servable `Kinds` (at minimum: + `Deployment, ReplicaSet, Pod, Rollout, Node, Service, Namespace, Event`). + +**`Read(ctx, ref)` — one resource:** + +1. Select the client for `ref.Scope` (the context). Fail-safe if the scope is unknown/unreachable + → typed error, not a panic. +2. `GET` the object by `ref.Kind`/`Namespace`/`Name` via client-go, using `ref.Attributes["gvr"]` + to resolve the group-version-resource where needed. +3. Normalize into `Evidence`: set `Kind` (usually `FactInventory`, or `FactHealth` for a derived + condition), `Observed` = the normalized JSON, `ObservedAt` = read time, `Source` = context + name, `Provenance{Adapter:"local-kubeconfig", NativeID: uid}`. No `DeepLink` (local mode). + +**`Query(ctx, q)` — list resources across contexts (the cross-cluster read):** + +1. Determine target scopes: `q.Scopes` if set, else **all reachable** contexts from the last + discovery. +2. **Fan out concurrently, one independent read session per context.** Each session uses that + context's own client/identity. Sessions are independent: a slow or failing context degrades + *its* row, not the whole answer. +3. Apply the typed `q.Selector` per context (translate to a client-go `List` with label/field + selectors; filter `Health`/`Image` post-list where the API can't express it). +4. Normalize each result into a `Fact` (stamp `ObservedAt`, `Source`, `Workspace` = local ws). +5. Assemble `QueryResult{Facts, Coverage}` where **`Coverage` names every unreachable and every + stale scope** — the answer is honest about what it could not see (F2.5). Never silently drop a + context. + +**Cache-first note (Slice 2 hook, not Slice 1).** F11.2's <100ms render comes from informer/watch +sessions hydrating a local store that `Query` reads first. Slice 1 may implement `Query` as direct +`List` calls; the interface is identical, so Slice 2 swaps the backing store without changing the +contract. Specify the seam; don't force the cache into Slice 1. + +### 5.3 Providing the Slice-0 `fleet.Source` (§7 bridge in action) + +Slice-0's `sith clusters` depends on `fleet.Source.Fleet(ctx) (FleetResult, error)`. The +kubeconfig adapter satisfies it **through the bridge**, not by re-implementing snapshot logic: +`Fleet()` = `Discover()` (→ `Cluster` rows) composed with a standing inventory/health `Query` +(→ populate status), assembled into `FleetResult` with the same `Coverage`. Slice 1's only CLI +change is the one injection line Slice-0 §3.5 already isolated: `fleet.StubSource{}` → +`connector.AsSource(kubeconfigConnector)`. + +--- + +## 6. Staying additive — why W1–W4 connectors are cheap + +Adding any integration is: **(1)** a new package under `internal/connector//`; **(2)** +implement the capability interfaces it supports; **(3)** `Register` a factory. **Zero** changes +above the seam — not the CLI, not the fleet model, not the query engine, not the MCP tools. This +is the E12/F12 discipline expressed at the type level: the taxonomy is closed (three kinds), the +protocol is additive (minor-version), and one canonical connector owns each tool. + +| Wave | Tool | Connector kind | Capabilities (interfaces) | Facts / verbs | +|---|---|---|---|---| +| W1 | Kubernetes (kubeconfig) | read-adapter | discover/read/query (`Reader`) | inventory, health | +| W1 | Kubernetes (OCM spoke) | read-adapter | discover/read/query (`Reader`) | inventory, health (hub mode) | +| W1 | GitHub | read-adapter + typed-action | read/query (`Reader`) + plan/execute/verify (`Planner`/`Executor`/`Verifier`) | drift/PR facts; verb `gitops.open-pr` | +| W1 | ArgoCD | typed-action | full contract (`Reader`+`Differ`+`Planner`+`Executor`+`Verifier`) | drift, health; verbs `argocd.sync`, `argocd.rollback` | +| W1 | Prometheus | read-adapter | discover/read/query (`Reader`) | health, alert | +| W1 | Elasticsearch | read-adapter | discover/read/query (`Reader`) | alert, inventory (log-derived) | +| W1 | AWS | read-adapter | discover/read/query (`Reader`) | inventory, cost inputs | +| W2 | Helm | read-adapter | read/query (`Reader`) | inventory (releases) | +| W2 | Kustomize | read-adapter | diff (`Differ`) | drift | +| W2 | kubectl-diff | read-adapter | diff (`Differ`) | drift | +| W3 | Grafana | **brokered-read-through** | discover/read (`Reader`, deep-link only) | health (via `Provenance.DeepLink` — never re-skinned) | +| W3 | OTel | read-adapter | read/query (`Reader`) | health, alert | +| W3 | OpenShift | read-adapter | discover/read/query (`Reader`) | inventory, health | +| W3 | Azure | read-adapter | discover/read/query (`Reader`) | inventory, cost inputs | +| W3 | GCP | read-adapter | discover/read/query (`Reader`) | inventory, cost inputs | +| W4 | OpenSearch | read-adapter | read/query (`Reader`) | alert, inventory | +| W4 | Splunk | read-adapter | read/query (`Reader`) | alert | +| W4 | Fluentd / FluentBit | read-adapter | discover/read (`Reader`) | inventory (pipeline state) | +| W4 | Istio / Linkerd | read-adapter | discover/read/query (`Reader`) | inventory, health (mesh) | +| W4 | Docker | read-adapter | discover/read/query (`Reader`) | inventory (containers/images) | + +Three observations that prove the seam holds: + +1. **Read-only tools implement one small interface.** Most of W1–W4 are `Reader`-only. They never + touch the write path, so they carry none of its risk or ceremony. +2. **Grafana is a *kind*, not a special case.** "Deep-link, don't re-skin" is expressed by + `KindBrokeredRead` + `Provenance.DeepLink` — the devops-portal iframe trap is not an expressible + connector shape (F12.2 guardrail). +3. **Write tools opt into exactly the verbs they host.** ArgoCD declares + `CapExecute` + `Verbs:["argocd.sync","argocd.rollback"]`; the registry refuses an executor that + doesn't map to the closed vocabulary. Adding `argocd.rollback` later is additive within the + same connector — no new architecture. + +E12 later moves these out-of-process over gRPC (F12.1); the **contract is identical** — the gRPC +SDK marshals these same verbs and types. In-process (day-1 six) vs. out-of-process (E12) is a +transport swap beneath an unchanged contract, which is exactly why the day-1 adapters port onto +the framework without behavior change (F12.4). + +--- + +## 7. The `fleet.Source ⇄ connector.Reader` bridge (don't redefine Slice 0) + +Slice-0's `fleet.Source` is the **snapshot** seam (`Kind()` + `Fleet(ctx) FleetResult`). The +connector contract is the **verb** seam beneath it. They meet through a one-way adapter that lives +in `connector` (so `fleet` stays dependency-free — §1): + +```go +package connector + +// AsSource adapts any Reader to the Slice-0 fleet.Source snapshot seam, so the +// existing `sith clusters` wiring consumes a real connector unchanged. Fleet() +// composes Discover (cluster rows) with a standing inventory/health Query (status), +// preserving Coverage so a partial view stays honest. +func AsSource(r Reader) fleet.Source +``` + +Consequences: + +- **Slice 0 is untouched.** Its `Source` interface, `FleetResult`, `Cluster`, `Coverage`, + `StubSource`, and the `clusters` render code all stay exactly as specified. Slice 1 changes the + single injection line and adds packages — nothing in `cmd/` or `internal/cli/` moves. +- **The hub reuses the identical bridge.** The OCM-spoke read adapter (#9) is another `Reader`; + `AsSource` turns it into a `fleet.Source` too. Local and hub are one code path above the source, + exactly as E2/E11 require. +- **Consumers pick their altitude.** `sith clusters` wants a snapshot → uses `fleet.Source`. The + cross-cluster query engine and MCP read tools want facts → use `Reader.Query` directly. Same + connector underneath. + +--- + +## 8. Acceptance criteria + +### 8.1 Contract (this spec's deliverable — types & registry) + +- [ ] `internal/fleet` gains `ResourceRef`, `FactKind`, `Evidence`, `Provenance`, `Fact`, `Query`, + `Selector`, `QueryResult`, `Diff`/`DiffHunk`, `Graph`/`Node`/`Edge`/`Relation` — additively, + without changing the Slice-0 `Source`/`FleetResult`/`Cluster` signatures. +- [ ] `Coverage` extended with `Stale` and a `Complete()` method; `FleetResult` unchanged. +- [ ] `internal/connector` defines `Connector`, `Reader`, `Differ`, `Planner`, `Executor`, + `Verifier`, `Descriptor`, `ConnectorKind`, `Capability`, and the supporting `Intent`, + `ActionPlan`, `PlanStep`, `ExecutionResult`, `VerifyRequest`, `Verification`, `DiffRequest`, + `Discovery`, `Scope`. +- [ ] `Registry` enforces, with tests: (a) one canonical connector per `Kind`; (b) `ConnKind` ∈ + the three; (c) **every declared capability's interface is implemented** (declared-but-unimplemented + → registration error); (d) typed-action connectors declare closed-vocab `Verbs`. +- [ ] `WithCapability` / `ReaderFor` / `ExecutorFor` return only connectors that **both declare and + implement** the capability; a read adapter can never be obtained as an `Executor`. +- [ ] The dependency direction `kubeconfig → connector → fleet` holds with no import cycle + (a CI `go list`/`depguard` check is acceptable). + +### 8.2 First adapter (Slice 1 — local-kubeconfig) + +- [ ] `local-kubeconfig` implements `Reader` (discover/read/query) and **not** the write + interfaces; `Capabilities()` == `{discover, read, query}`. +- [ ] `Discover` enumerates **all** kubeconfig contexts from `$KUBECONFIG`/`~/.kube/config`; + reachable contexts stream, **unreachable contexts are surfaced in `Discovery.Unreachable`**, + never fatal; probes are concurrent and per-context time-boxed. +- [ ] Exec-credential plugins run **locally**; **no credential or kubeconfig is copied off the + machine** (seeded assertion for the Slice-5 egress test). +- [ ] `Query` fans out across contexts concurrently with independent sessions and returns + `Coverage` that names every unreachable and stale scope; a single query answers over + ≥ 2 contexts. +- [ ] `connector.AsSource(local-kubeconfig)` satisfies `fleet.Source`; `sith clusters` lists + **real** contexts with unreachable ones flagged, changing exactly **one** injection line in + the CLI (Slice-0 §3.5). +- [ ] A second in-memory `Reader` in tests registers and flows through the same `Registry` and + `AsSource` path — proving the seam admits multiple adapters (the additive guarantee). + +### 8.3 Additivity (the seam's whole point) + +- [ ] A hello-world read connector (returns one fake `Fact`) can be added as **one new package + + one `Register` call**, with **no change** to `internal/fleet`, `internal/cli`, or the query + engine — asserted by a test that registers it and reads its fact back through `WithCapability`. + +--- + +## 9. Open decisions (recorded, not blocking) + +1. **`Observed` typing — `json.RawMessage` vs. a typed union.** Chosen: `json.RawMessage`, decoded + by `FactKind`, to keep `fleet` dependency-free and additive. Revisit if a typed `oneof`/generics + model proves worth the coupling once ≥ 3 fact kinds exist. *Default stands; not a Slice-1 + blocker.* +2. **Graph assembly ownership.** Specified in the model; **built in Slice 2** (F11.4/F2.3), not + Slice 1. Slice 1 emits facts + obvious ownership hints only. Open: whether adapters ever emit + cross-source edges (e.g. Prometheus alert → K8s Deployment) or the model joins them by + `ResourceRef`. Leaning: **model joins**; adapters stay single-source. *Decide in Slice 2.* +3. **Reachability probe cost.** `/version` vs. `SelfSubjectAccessReview` vs. lazy (probe on first + real read). Leaning: cheap `/version` at `Discover`, lazy upgrade on `Read`/`Query`. *Tunable; + not contract-affecting.* +4. **`ProtocolV` granularity.** One protocol version for the whole contract vs. per-verb. Chosen: + one per connector (F12.3 semver), simplest that satisfies "minor is additive". *Revisit at E12 + when gRPC wire format lands.* +5. **Where `Workspace` is stamped in local mode.** A single implicit local workspace constant + vs. threading E1's type down. Leaning: a `fleet.LocalWorkspace` constant so the type is present + from day 0 and the hub swap is mechanical. *Confirm against E1's final Workspace type.* +6. **Cross-source `ResourceRef` identity.** `Equal` ignores `Attributes`; is `(SourceKind, Scope, + Kind, Namespace, Name)` enough to collide the *same* workload seen via two connectors (kubeconfig + *and* ArgoCD)? Possibly needs a normalized `UID`/`fingerprint`. *Defer until the second connector + that observes shared resources (ArgoCD, W1) forces the question.* + +--- + +## 10. What this spec deliberately does not do + +- It does **not** implement anything — no product Go beyond the reference signatures here. +- It does **not** redefine Slice-0's `fleet.Source`, `FleetResult`, `Cluster`, or `StubSource`. +- It does **not** build the write path (diff/plan/execute/verify bodies), the operational-graph + assembler, the informer/watch cache, the OCM-spoke adapter, gRPC out-of-process transport, or + any W1–W4 connector beyond specifying where each slots in. +- It does **not** introduce a database, telemetry, or any non-loopback listener — the trust + invariants (`CONVENTIONS.md` §7) hold from here forward. + + diff --git a/docs/specs/SLICE-0-foundation.md b/docs/specs/SLICE-0-foundation.md new file mode 100644 index 0000000..c4aee63 --- /dev/null +++ b/docs/specs/SLICE-0-foundation.md @@ -0,0 +1,602 @@ +# Spec — Slice 0: Foundation Walking-Skeleton + +**Status:** ready to build · **Date:** 2026-07-10 · **Target builder:** a fresh Sonnet session, max effort +**Branch:** `feat/slice-0-foundation` off `dev` · **PR target:** `dev` +**Prereqs to read:** [`../CONVENTIONS.md`](../CONVENTIONS.md) (commits, CI gates, style), +[`../BUILD-SEQUENCE.md`](../BUILD-SEQUENCE.md) (where this slice sits), +[`../adr/0002-stack-and-language.md`](../adr/0002-stack-and-language.md) (Go, single binary). + +This spec is **self-contained**: build exactly what is written here, nothing more. Do not implement +kubeconfig fan-out, real clusters, TUI, web UI, MCP, or keychain — those are Slices 1–6. Slice 0 is +the scaffold plus one end-to-end path through a **stubbed** fleet source. All disk work stays on +`/Volumes/EXTENDED` (never the system disk). + +--- + +## 0. Definition of done (read this first) + +A reviewer merges this PR when **all** of the following are true: + +1. `make build` produces `bin/sith` with version metadata injected via ldflags. +2. `bin/sith version` prints build info (text) and `bin/sith version -o json` prints valid JSON. +3. `bin/sith clusters` calls a **stubbed `fleet.Source`**, receives an **empty typed `FleetResult`**, + and prints a clean "no clusters" result; `-o json` prints `{"clusters":[],"coverage":{…}}`. Exit 0. +4. `bin/sith ui` and `bin/sith hub` print a clear "not yet implemented — see " line, exit 0. +5. `bin/sith` (no args) and `bin/sith --help` print usage, exit 0. An unknown subcommand exits non-zero. +6. `make ci` is green locally: `fmt-check`, `vet`, `lint`, `test -race`, `build`. +7. GitHub Actions CI is green on the PR (same gates). +8. Every package in §3 has the tests listed in §7, and they pass under `-race`. +9. `sessions/` already has `README.md` + `JOURNAL-TEMPLATE.md` (from the plan branch); this slice + **adds** `sessions/2026-07-
-slice-0-foundation.md` with a `[C]` checkpoint per commit and + matching `GSTACK-Checkpoint` trailers. +10. All commits are Conventional + DCO signed-off + SSH-signed, **no AI attribution anywhere** + (`CONVENTIONS.md` §2). No `main` branch changes. + +Slice 0 is **independent of open questions Q12–Q15** — do not let any of them influence this slice. + +--- + +## 1. Module & toolchain + +- **Module path:** `github.com/ArdurAI/sith` +- **Go version:** `go 1.24` (the `go` directive in `go.mod`). CI pins `1.24.x`. +- **Init:** + ```bash + cd /Volumes/EXTENDED/repos/sith + go mod init github.com/ArdurAI/sith # if go.mod absent + ``` +- **Dependencies (resolve exact versions with the toolchain, do not hand-pin):** + ```bash + go get github.com/spf13/cobra@latest # CLI framework + go get gopkg.in/yaml.v3@latest # config file parsing + go mod tidy + go mod verify + ``` + Everything else is the standard library (`log/slog`, `context`, `encoding/json`, `runtime`, + `os`, `flag`-free — cobra owns flags). + +### CLI framework: **cobra** (decision, justified) + +Use **`github.com/spf13/cobra`**. Rationale, briefly: it is the de-facto standard for Go +operational CLIs — `kubectl`, `helm`, `k9s`, `argo`, and `gh` all use it — so it matches the +mental model of Sith's exact audience; it gives a clean subcommand tree, POSIX flags (via `pflag`), +built-in help, and shell completion for free; and it scales to the `sith`/`sith ui`/`sith hub`/ +`sith serve --mcp` tree without rework. **Not viper:** Slice 0's config needs are tiny, and viper +pulls a large transitive tree that inflates the SBOM and supply-chain surface (a first-order +concern for a cosign/SLSA/SBOM project, E9). We hand-roll a ~60-line config loader over `yaml.v3` +instead (§ `internal/config`); revisit viper only if config genuinely grows. **Not urfave/cli or +stdlib `flag`:** less ecosystem alignment and a thinner subcommand/ completion story for the tree we +need. + +--- + +## 2. Directory & file layout (create exactly this) + +``` +sith/ +├── go.mod # module + go 1.24 + require cobra, yaml.v3 +├── go.sum +├── Makefile # §5 +├── .golangci.yml # CONVENTIONS.md §4.2 (v2 schema) — copy verbatim +├── .gitignore # extend existing: add /bin/, *.out, coverage.* +├── .github/ +│ └── workflows/ +│ └── ci.yml # §6 +├── cmd/ +│ └── sith/ +│ └── main.go # tiny entrypoint → internal/cli.Execute() +├── internal/ +│ ├── buildinfo/ +│ │ ├── buildinfo.go # Version/Commit/Date vars (ldflags) + String()/JSON() +│ │ └── buildinfo_test.go +│ ├── config/ +│ │ ├── config.go # Config struct + Load() precedence loader +│ │ └── config_test.go +│ ├── logging/ +│ │ ├── logging.go # slog logger builder +│ │ └── logging_test.go +│ ├── fleet/ +│ │ ├── model.go # typed fleet model: Cluster, FleetResult, Coverage +│ │ ├── source.go # the Source interface — the F2.1 seam (#38) +│ │ ├── stub.go # StubSource: returns an empty FleetResult +│ │ └── fleet_test.go +│ └── cli/ +│ ├── root.go # root cmd, persistent flags, wiring, Execute() +│ ├── version.go # `sith version` +│ ├── clusters.go # `sith clusters` (uses fleet.Source) +│ ├── ui.go # `sith ui` stub +│ ├── hub.go # `sith hub` stub +│ └── cli_test.go +├── sessions/ # scaffold ALREADY EXISTS (from the docs/build-plan branch) +│ ├── README.md # present — do not recreate +│ ├── JOURNAL-TEMPLATE.md # present — copy it to start your session +│ └── 2026-07-
-slice-0-foundation.md # ADD this: your session's journal +└── docs/ … # already present; do not modify plan docs +``` + +Every `.go` file starts with `// SPDX-License-Identifier: Apache-2.0` on line 1, then the package +clause. Package name == directory name. + +--- + +## 3. Package specs (signatures + behavior — the builder implements the bodies) + +> These are the **contracts**. Implement idiomatic Go bodies that satisfy them and the tests in §7. +> Snippets are reference signatures, not finished code. + +### 3.1 `internal/buildinfo` + +Injected via `-ldflags -X` (see Makefile). Defaults must be self-describing when built without +ldflags (e.g. `go run`). + +```go +package buildinfo + +// These are overwritten at build time via -ldflags -X. Keep the defaults. +var ( + Version = "dev" + Commit = "none" + Date = "unknown" +) + +// Info is the resolved build metadata, including runtime-derived fields. +type Info struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + Go string `json:"go"` // runtime.Version() + Platform string `json:"platform"` // runtime.GOOS + "/" + runtime.GOARCH +} + +func Get() Info // fills Version/Commit/Date + Go/Platform from runtime +func (i Info) String() string // multi-line human text (see §4 output format) +func (i Info) JSON() (string, error) // compact JSON +``` + +### 3.2 `internal/config` + +Hand-rolled precedence loader. **Precedence (lowest→highest): defaults → config file → env → flags.** + +```go +package config + +type Config struct { + LogLevel string `yaml:"log_level"` // debug|info|warn|error (default: info) + LogFormat string `yaml:"log_format"` // text|json (default: text) + KubeconfigPath string `yaml:"kubeconfig_path"` // reserved for Slice 1; unused in Slice 0 +} + +func Defaults() Config + +// Load resolves config from (in precedence order) defaults, the YAML file at `path` +// (or the default location if path==""), then env vars, then any non-zero Overrides. +// Env vars: SITH_LOG_LEVEL, SITH_LOG_FORMAT, SITH_KUBECONFIG. +// Default file location: $XDG_CONFIG_HOME/sith/config.yaml, else ~/.config/sith/config.yaml. +// A missing default-location file is NOT an error; a missing explicitly-passed path IS an error. +// Validate() runs at the end: unknown LogLevel/LogFormat -> error (fail-safe, never default-open). +func Load(path string, overrides Overrides) (Config, error) + +type Overrides struct { // non-empty fields win (from flags) + LogLevel string + LogFormat string +} + +func (c Config) Validate() error +``` + +Fail-safe (`CONVENTIONS.md` §7.5): an invalid `log_level`/`log_format` returns an error; it is never +silently coerced. + +### 3.3 `internal/logging` + +```go +package logging + +// New builds a *slog.Logger for the given level/format writing to w (usually os.Stderr). +// format: "text" -> slog.NewTextHandler; "json" -> slog.NewJSONHandler. +// level: debug|info|warn|error -> slog.Level. Unknown level/format -> error. +func New(w io.Writer, level, format string) (*slog.Logger, error) +``` + +User-facing command output (version text, cluster tables) goes to **stdout via `fmt`**; diagnostics +and structured logs go to the **slog logger (stderr)**. Never mix the two. + +### 3.4 `internal/fleet` — the F2.1 seam (the load-bearing part of Slice 0) + +This is the interface Slice 1 (F2.1 #38) implements with a real local-kubeconfig adapter and the +hub later implements with an OCM-spoke adapter. Slice 0 ships the **types + interface + a stub**. + +```go +package fleet + +import ( + "context" + "time" +) + +// Source is the read seam every fleet backend implements. Day-0 local mode provides a +// local-kubeconfig adapter (F2.1/#38); day-N hub provides an OCM-spoke adapter (#9). +// Everything above this interface is shared, one code path over many sources. +type Source interface { + // Kind identifies the adapter, e.g. "stub", "local-kubeconfig", "ocm-spoke". + Kind() string + // Fleet returns the current normalized fleet snapshot for this source. + Fleet(ctx context.Context) (FleetResult, error) +} + +// FleetResult is the normalized snapshot returned by a Source. +type FleetResult struct { + Clusters []Cluster `json:"clusters"` + // Coverage makes a partial view impossible to mistake for a complete one + // (SITH-NOTION.md F2.5): what was asked for, what answered, what did not. + Coverage Coverage `json:"coverage"` +} + +// Cluster is one cluster/context in the fleet, freshness- and source-stamped (F2.2). +type Cluster struct { + Name string `json:"name"` // display name (context name in local mode) + Context string `json:"context,omitempty"` // kubeconfig context (local mode) + SourceKind string `json:"source_kind"` // the Source.Kind() that produced it + Reachable bool `json:"reachable"` + ObservedAt time.Time `json:"observed_at,omitempty"` // zero => never observed +} + +// Coverage summarizes reachability across the queried sources/contexts. +type Coverage struct { + Requested int `json:"requested"` + Reachable int `json:"reachable"` + Unreachable []string `json:"unreachable,omitempty"` // names/contexts that failed +} + +// StubSource returns an empty, well-formed FleetResult. It exists so the CLI has a +// complete end-to-end path in Slice 0; Slice 1 replaces it with the local-kubeconfig adapter. +type StubSource struct{} + +func (StubSource) Kind() string { return "stub" } +func (StubSource) Fleet(ctx context.Context) (FleetResult, error) { + return FleetResult{Clusters: []Cluster{}, Coverage: Coverage{}}, nil +} +``` + +**Do not** add kubeconfig, client-go, or informer logic in Slice 0. The stub is the whole backend. +Keeping the interface minimal-but-correct here is the point: Slice 1 slots in without touching +`cmd/` or `internal/cli/`. + +### 3.5 `internal/cli` + +Root command + subcommands, using cobra. `Execute()` is the single entrypoint `main` calls. + +```go +package cli + +// Execute builds the root command and runs it; returns a process exit code. +func Execute() int +``` + +Root command wiring: +- `Use: "sith"`, short/long descriptions naming ArdurAI and the local fleet client. +- **Persistent flags:** `--log-level` (default from config), `--log-format`, `--config` (path), + `-o, --output` (`text`|`json`, default `text`). +- `PersistentPreRunE`: load config (`config.Load`) with flag overrides, build the slog logger + (`logging.New`), stash both in the command context. Any config/logger error aborts with a clear + message and non-zero exit. +- A single `fleet.Source` is constructed once (Slice 0: `fleet.StubSource{}`) and injected into the + `clusters` command. Keep this injectable (a package-level var or a field) so Slice 1 swaps the + stub for the real adapter by changing **one** line. +- `SilenceUsage: true` and `SilenceErrors: true` on the root; handle/print errors in `Execute()` so + usage noise does not print on runtime errors, and map errors to a non-zero code. + +Subcommand behavior (exact, for deterministic tests): + +| Command | stdout (text mode) | JSON mode (`-o json`) | Exit | +|---|---|---|---| +| `sith version` | see §4 | `{"version":…,"commit":…,"date":…,"go":…,"platform":…}` | 0 | +| `sith clusters` (empty) | `No clusters found (source: stub — F2.1/#38 not yet implemented).` | `{"clusters":[],"coverage":{"requested":0,"reachable":0}}` | 0 | +| `sith ui` | `sith ui: not yet implemented — see F11.3 (#34).` | same line | 0 | +| `sith hub` | `sith hub: not yet implemented — hub mode is phase-1+ (E1–E10).` | same line | 0 | +| `sith` / `sith --help` | cobra usage | — | 0 | +| unknown subcommand | cobra "unknown command" error to stderr | — | **non-zero** | + +`clusters` renders from whatever the injected `Source.Fleet(ctx)` returns; in Slice 0 that is always +empty, but the render code must handle a non-empty result generically (a simple aligned table: +`NAME CONTEXT SOURCE REACHABLE OBSERVED`) so Slice 1 needs no CLI change. + +### 3.6 `cmd/sith/main.go` + +```go +package main + +import ( + "os" + + "github.com/ArdurAI/sith/internal/cli" +) + +func main() { os.Exit(cli.Execute()) } +``` + +--- + +## 4. `sith version` output formats (exact) + +**Text (default):** +``` +sith + commit: + built: + go: + platform: / +``` + +**JSON (`-o json`):** a single compact line, e.g. +```json +{"version":"dev","commit":"none","date":"unknown","go":"go1.24.4","platform":"darwin/arm64"} +``` + +Text goes to stdout. JSON goes to stdout. Nothing else on stdout for these commands. + +--- + +## 5. `Makefile` (create verbatim) + +```makefile +# Sith — Makefile +SHELL := /usr/bin/env bash + +BINARY := sith +PKG := github.com/ArdurAI/sith +CMD := ./cmd/sith +BIN_DIR := bin +GOLANGCI := golangci-lint + +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +LDFLAGS := -s -w \ + -X $(PKG)/internal/buildinfo.Version=$(VERSION) \ + -X $(PKG)/internal/buildinfo.Commit=$(COMMIT) \ + -X $(PKG)/internal/buildinfo.Date=$(DATE) + +.PHONY: all build test lint fmt fmt-check vet tidy clean run ci help + +all: build + +build: ## Build the sith binary into bin/ + @mkdir -p $(BIN_DIR) + go build -trimpath -ldflags '$(LDFLAGS)' -o $(BIN_DIR)/$(BINARY) $(CMD) + +test: ## Run unit tests with the race detector + go test -race -count=1 ./... + +lint: ## Run golangci-lint (v2) + $(GOLANGCI) run ./... + +fmt: ## Format code (gofmt + goimports via golangci-lint v2 formatters) + $(GOLANGCI) fmt ./... + +fmt-check: ## Fail if formatting/imports would change anything + gofmt -l . | tee /dev/stderr | (! read) + $(GOLANGCI) fmt --diff ./... + +vet: ## go vet + go vet ./... + +tidy: ## Tidy and verify modules + go mod tidy + go mod verify + +clean: ## Remove build artifacts + rm -rf $(BIN_DIR) + +run: build ## Build then run `sith version` + $(BIN_DIR)/$(BINARY) version + +ci: fmt-check vet lint test build ## Run the full CI gate locally + +help: ## List targets + @grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | \ + awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-10s\033[0m %s\n",$$1,$$2}' +``` + +`make fmt`/`fmt-check`/`lint` require golangci-lint v2 on PATH (install: +`https://golangci-lint.run/welcome/install/`, pin the same v2.x as CI). `gofmt` is bundled with Go. + +--- + +## 6. GitHub Actions CI — `.github/workflows/ci.yml` (create verbatim) + +Must go green. Runs on PRs into `dev` and pushes to `dev`. + +```yaml +name: ci + +on: + push: + branches: [dev] + pull_request: + branches: [dev] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: "1.24.x" + GOLANGCI_VERSION: "v2.1.6" # pin; bump is its own ci: commit + +jobs: + build-test-lint: + name: build · vet · gofmt · lint · test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: Download & verify modules + run: | + go mod download + go mod verify + + - name: gofmt check + run: | + diff=$(gofmt -l .) + if [ -n "$diff" ]; then + echo "::error::gofmt needed on:"; echo "$diff"; exit 1 + fi + + - name: go vet + run: go vet ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@v7 + with: + version: ${{ env.GOLANGCI_VERSION }} + args: run ./... + + - name: format & imports check (golangci-lint fmt) + run: golangci-lint fmt --diff ./... + + - name: build + run: go build -trimpath ./... + + - name: test (race) + run: go test -race -count=1 ./... +``` + +> The `golangci/golangci-lint-action@v7` step installs golangci-lint on PATH for the subsequent +> `golangci-lint fmt --diff` step. If a builder finds the binary is not on PATH in that step, +> install it explicitly via the official install script pinned to `${GOLANGCI_VERSION}` before the +> fmt step — do not drop the fmt/imports gate. +> +> A **DCO check** and **signed-commit / review** requirements are enforced by branch protection on +> `dev` (`CONVENTIONS.md` §5), configured in the repo settings — not in this workflow file. + +Copy `.golangci.yml` verbatim from `CONVENTIONS.md` §4.2 (golangci-lint **v2** schema). Do not +weaken it to pass; write code that satisfies it. + +--- + +## 7. Test scaffolding — the exact test list + +All tests are hermetic (no network, no clusters) and pass under `go test -race`. Use table-driven +style where noted. Test cobra commands **in-process** by constructing the root command and setting +its args + capturing its output (do not spawn the binary in unit tests). + +### `internal/buildinfo/buildinfo_test.go` +- `TestGetPopulatesRuntimeFields` — `Get().Go` and `.Platform` are non-empty and match + `runtime.Version()` / `GOOS/GOARCH`. +- `TestStringContainsAllFields` — `Info.String()` contains version, commit, date, go, platform. +- `TestJSONRoundTrips` — `Info.JSON()` unmarshals back into an equal `Info`. + +### `internal/config/config_test.go` +- `TestDefaults` — `Defaults()` == `{info, text, ""}`. +- `TestLoadFromFile` — a temp YAML file sets level/format; `Load` reflects it. +- `TestEnvOverridesFile` — `SITH_LOG_LEVEL`/`SITH_LOG_FORMAT` override file values. +- `TestOverridesBeatEnv` — non-empty `Overrides` (flags) beat env. +- `TestMissingDefaultFileIsOK` — no default-location file → no error, defaults returned. +- `TestExplicitMissingPathErrors` — a non-existent explicitly-passed `--config` path → error. +- `TestInvalidLevelRejected` / `TestInvalidFormatRejected` — fail-safe: invalid values → error + (table-driven). + +### `internal/logging/logging_test.go` +- `TestNewTextHandler` — format `text` produces a logger; a logged line is plain text (not JSON). +- `TestNewJSONHandler` — format `json` produces valid JSON lines (parse a captured line). +- `TestLevelFiltering` — at level `warn`, an `Info` line is suppressed and a `Warn` line emitted. +- `TestInvalidLevelOrFormatErrors` — unknown level/format → error (table-driven). + +### `internal/fleet/fleet_test.go` +- `TestStubSourceKind` — `StubSource{}.Kind() == "stub"`. +- `TestStubSourceEmpty` — `Fleet(ctx)` returns zero clusters and zero coverage, no error. +- `TestSourceInterfaceSatisfied` — compile-time `var _ Source = StubSource{}` plus a second + trivial in-memory `Source` (declared in the test) to prove the interface admits multiple + adapters (the F2.1/OCM-parity guarantee) and flows through the same result type. +- `TestFleetResultJSONShape` — marshaling an empty `FleetResult` yields + `{"clusters":[],"coverage":{"requested":0,"reachable":0}}` (asserts the CLI JSON contract). + +### `internal/cli/cli_test.go` +- `TestVersionText` — `sith version` stdout contains `sith ` and the platform. +- `TestVersionJSON` — `sith version -o json` stdout parses as JSON with all five keys. +- `TestClustersEmptyText` — `sith clusters` stdout is the "No clusters found" line, exit 0. +- `TestClustersEmptyJSON` — `sith clusters -o json` parses to an empty clusters array. +- `TestClustersUsesInjectedSource` — inject a fake `Source` returning 2 clusters; assert the table + renders both rows (proves Slice 1 needs no CLI change). +- `TestUIStub` / `TestHubStub` — the exact stub lines, exit 0. +- `TestRootHelpExitsZero` — `sith --help` exits 0 and prints usage. +- `TestUnknownCommandNonZero` — `sith bogus` returns a non-zero code. +- `TestInvalidLogLevelFlagFails` — `--log-level nope` aborts non-zero (config fail-safe reaches the + CLI). + +**Optional e2e (behind `//go:build e2e`, not in the default gate):** `TestBinarySmoke` builds and +runs `bin/sith version` + `bin/sith clusters`, asserting output. Add a `make e2e` target if you +include it. Not required for merge. + +--- + +## 8. Build & verify runbook (what the builder runs) + +```bash +cd /Volumes/EXTENDED/repos/sith +git checkout -b feat/slice-0-foundation dev + +# ... create files per §2–§7 ... + +go mod tidy && go mod verify +make fmt # normalize +make ci # fmt-check + vet + lint + test -race + build → must be all green +./bin/sith version +./bin/sith version -o json +./bin/sith clusters +./bin/sith clusters -o json +./bin/sith ui +./bin/sith hub +./bin/sith --help +./bin/sith bogus; echo "exit=$?" # expect non-zero +``` + +Commit in small signed increments (suggested checkpoints, each a `[C]` journal entry + matching +`GSTACK-Checkpoint` trailer): +1. `chore(build): go module, gitignore, Makefile, .golangci.yml` +2. `feat(buildinfo): build metadata + version formatting` +3. `feat(config): precedence config loader with fail-safe validation` +4. `feat(logging): slog logger builder` +5. `feat(fleet): source-abstract fleet model + stub source (F2.1 seam, #38)` +6. `feat(cli): root command + version/clusters/ui/hub with cobra` +7. `ci(actions): build/vet/gofmt/lint/test workflow` +8. `docs(sessions): GSTACK journal for slice 0` + +Open the PR into `dev`, confirm CI green, and stop (do not merge — the owner reviews). + +--- + +## 9. Acceptance criteria (restated, checkable) + +- [ ] `make ci` green locally; GitHub Actions CI green on the PR. +- [ ] `sith version` (text + json) prints injected build metadata. +- [ ] `sith clusters` returns a typed empty `FleetResult` from `fleet.StubSource`; text + json render. +- [ ] `sith ui` / `sith hub` print their stub lines, exit 0; unknown command exits non-zero. +- [ ] `fleet.Source` interface exists exactly as §3.4; a second in-memory adapter satisfies it in + tests (F2.1 seam proven). +- [ ] All §7 tests present and passing under `-race`. +- [ ] `sessions/` scaffold + this session's journal committed; checkpoints ↔ commit trailers. +- [ ] Commits Conventional + DCO + SSH-signed; **no AI attribution**; `main` untouched. +- [ ] No kubeconfig/client-go/TUI/web/MCP/keychain code — those are Slices 1–6. + +--- + +## 10. Explicit non-goals for Slice 0 (do not build) + +Kubeconfig discovery or fan-out (Slice 1) · client-go / informers (Slice 1) · any real cluster read +· the TUI (Slice 2) · cross-cluster search/correlation (Slice 2) · per-pod logs/exec/port-forward/ +YAML (Slice 3) · `sith ui` web server or embedded frontend (Slice 4) · keychain / telemetry-egress +test (Slice 5) · MCP server / `sith serve --mcp` (Slice 6) · brew/goreleaser/cosign/SLSA/SBOM +(Slice P) · viper · a database · anything hub/OCM. If you are tempted, it belongs to a later slice — +leave the seam and stop. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..da62d11 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/ArdurAI/sith + +go 1.25.0 + +require ( + github.com/spf13/cobra v1.10.2 + go.yaml.in/yaml/v3 v3.0.4 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e63b363 --- /dev/null +++ b/go.sum @@ -0,0 +1,12 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go new file mode 100644 index 0000000..7fc0384 --- /dev/null +++ b/internal/buildinfo/buildinfo.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package buildinfo exposes build metadata injected by the build pipeline. +package buildinfo + +import ( + "encoding/json" + "fmt" + "runtime" +) + +// Version is the semantic version or development identifier injected at build time. +var Version = "dev" + +// Commit is the source commit injected at build time. +var Commit = "none" + +// Date is the UTC build timestamp injected at build time. +var Date = "unknown" + +// Info is the resolved build metadata, including runtime-derived fields. +type Info struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + Go string `json:"go"` + Platform string `json:"platform"` +} + +// Get returns the build metadata combined with the active Go runtime and platform. +func Get() Info { + return Info{ + Version: Version, + Commit: Commit, + Date: Date, + Go: runtime.Version(), + Platform: runtime.GOOS + "/" + runtime.GOARCH, + } +} + +// String returns build metadata in the human-readable CLI format. +func (i Info) String() string { + return fmt.Sprintf( + "sith %s\n commit: %s\n built: %s\n go: %s\n platform: %s", + i.Version, + i.Commit, + i.Date, + i.Go, + i.Platform, + ) +} + +// JSON returns build metadata as compact JSON. +func (i Info) JSON() (string, error) { + data, err := json.Marshal(i) + if err != nil { + return "", fmt.Errorf("marshal build metadata: %w", err) + } + + return string(data), nil +} diff --git a/internal/buildinfo/buildinfo_test.go b/internal/buildinfo/buildinfo_test.go new file mode 100644 index 0000000..1b85fba --- /dev/null +++ b/internal/buildinfo/buildinfo_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildinfo + +import ( + "encoding/json" + "runtime" + "strings" + "testing" +) + +func TestGetPopulatesRuntimeFields(t *testing.T) { + t.Parallel() + + got := Get() + if got.Go != runtime.Version() { + t.Fatalf("Go = %q, want %q", got.Go, runtime.Version()) + } + + wantPlatform := runtime.GOOS + "/" + runtime.GOARCH + if got.Platform != wantPlatform { + t.Fatalf("Platform = %q, want %q", got.Platform, wantPlatform) + } +} + +func TestStringContainsAllFields(t *testing.T) { + t.Parallel() + + info := Info{ + Version: "v1.2.3", + Commit: "abc1234", + Date: "2026-07-10T12:00:00Z", + Go: "go1.25.12", + Platform: "linux/amd64", + } + + for _, want := range []string{info.Version, info.Commit, info.Date, info.Go, info.Platform} { + if !strings.Contains(info.String(), want) { + t.Errorf("String() = %q, want it to contain %q", info.String(), want) + } + } +} + +func TestJSONRoundTrips(t *testing.T) { + t.Parallel() + + want := Info{ + Version: "v1.2.3", + Commit: "abc1234", + Date: "2026-07-10T12:00:00Z", + Go: "go1.25.12", + Platform: "linux/amd64", + } + + encoded, err := want.JSON() + if err != nil { + t.Fatalf("JSON() error = %v", err) + } + + var got Info + if err := json.Unmarshal([]byte(encoded), &got); err != nil { + t.Fatalf("unmarshal JSON: %v", err) + } + + if got != want { + t.Fatalf("round trip = %#v, want %#v", got, want) + } +} diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go new file mode 100644 index 0000000..e6d13e0 --- /dev/null +++ b/internal/cli/cli_test.go @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "runtime" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +type staticSource struct { + result fleet.FleetResult + called bool +} + +type failingSource struct{} + +func (failingSource) Kind() string { + return "failing" +} + +func (failingSource) Fleet(_ context.Context) (fleet.FleetResult, error) { + return fleet.FleetResult{}, errors.New("source unavailable") +} + +func (*staticSource) Kind() string { + return "memory" +} + +func (source *staticSource) Fleet(_ context.Context) (fleet.FleetResult, error) { + source.called = true + return source.result, nil +} + +func TestVersionText(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"version"}, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + if !strings.Contains(stdout, "sith ") || !strings.Contains(stdout, runtime.GOOS+"/"+runtime.GOARCH) { + t.Fatalf("stdout = %q, want version and platform", stdout) + } +} + +func TestVersionJSON(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"version", "-o", "json"}, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + + var got map[string]any + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("unmarshal stdout %q: %v", stdout, err) + } + for _, key := range []string{"version", "commit", "date", "go", "platform"} { + if _, ok := got[key]; !ok { + t.Errorf("JSON missing key %q: %#v", key, got) + } + } +} + +func TestClustersEmptyText(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"clusters"}, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + const want = "No clusters found (source: stub — F2.1/#38 not yet implemented).\n" + if stdout != want { + t.Fatalf("stdout = %q, want %q", stdout, want) + } +} + +func TestClustersEmptyJSON(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"clusters", "-o", "json"}, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + + var got fleet.FleetResult + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("unmarshal stdout %q: %v", stdout, err) + } + if got.Clusters == nil || len(got.Clusters) != 0 { + t.Fatalf("clusters = %#v, want allocated empty slice", got.Clusters) + } +} + +func TestClustersEmptyNonStubNamesSource(t *testing.T) { + source := &staticSource{result: fleet.FleetResult{}} + + stdout, _, exitCode := runCLI(t, []string{"clusters"}, source) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + if stdout != "No clusters found (source: memory).\n" { + t.Fatalf("stdout = %q, want dynamic source name", stdout) + } +} + +func TestClustersNormalizesNilSliceInJSON(t *testing.T) { + source := &staticSource{result: fleet.FleetResult{}} + + stdout, _, exitCode := runCLI(t, []string{"clusters", "-o", "json"}, source) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + const want = `{"clusters":[],"coverage":{"requested":0,"reachable":0}}` + "\n" + if stdout != want { + t.Fatalf("stdout = %q, want %q", stdout, want) + } +} + +func TestClustersSourceErrorIsConcise(t *testing.T) { + _, stderr, exitCode := runCLI(t, []string{"clusters"}, failingSource{}) + if exitCode == 0 { + t.Fatal("exit code = 0, want non-zero") + } + if !strings.Contains(stderr, "source unavailable") { + t.Fatalf("stderr = %q, want source error", stderr) + } + if strings.Contains(stderr, "Usage:") { + t.Fatalf("stderr = %q, want no usage noise", stderr) + } +} + +func TestClustersUsesInjectedSource(t *testing.T) { + source := &staticSource{result: fleet.FleetResult{ + Clusters: []fleet.Cluster{ + { + Name: "prod-us", + Context: "prod-us-admin", + SourceKind: "memory", + Reachable: true, + ObservedAt: time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC), + }, + {Name: "lab", Context: "lab", SourceKind: "memory"}, + }, + }} + + stdout, _, exitCode := runCLI(t, []string{"clusters"}, source) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0", exitCode) + } + if !source.called { + t.Fatal("injected source was not called") + } + for _, want := range []string{"NAME", "prod-us", "prod-us-admin", "lab", "memory"} { + if !strings.Contains(stdout, want) { + t.Errorf("stdout = %q, want it to contain %q", stdout, want) + } + } +} + +func TestUIStub(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"ui"}, fleet.StubSource{}) + if exitCode != 0 || stdout != "sith ui: not yet implemented — see F11.3 (#34).\n" { + t.Fatalf("exit/stdout = %d/%q", exitCode, stdout) + } +} + +func TestHubStub(t *testing.T) { + stdout, _, exitCode := runCLI(t, []string{"hub"}, fleet.StubSource{}) + if exitCode != 0 || stdout != "sith hub: not yet implemented — hub mode is phase-1+ (E1–E10).\n" { + t.Fatalf("exit/stdout = %d/%q", exitCode, stdout) + } +} + +func TestRootHelpExitsZero(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"--help"}, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0; stderr = %q", exitCode, stderr) + } + if !strings.Contains(stdout, "Usage:") || !strings.Contains(stdout, "sith [flags]") { + t.Fatalf("stdout = %q, want usage", stdout) + } +} + +func TestRootNoArgsExitsZero(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, nil, fleet.StubSource{}) + if exitCode != 0 { + t.Fatalf("exit code = %d, want 0; stderr = %q", exitCode, stderr) + } + if !strings.Contains(stdout, "Usage:") { + t.Fatalf("stdout = %q, want usage", stdout) + } +} + +func TestUnknownCommandNonZero(t *testing.T) { + _, stderr, exitCode := runCLI(t, []string{"bogus"}, fleet.StubSource{}) + if exitCode == 0 { + t.Fatal("exit code = 0, want non-zero") + } + if !strings.Contains(stderr, "unknown command") { + t.Fatalf("stderr = %q, want unknown command error", stderr) + } +} + +func TestInvalidLogLevelFlagFails(t *testing.T) { + _, stderr, exitCode := runCLI(t, []string{"clusters", "--log-level", "nope"}, fleet.StubSource{}) + if exitCode == 0 { + t.Fatal("exit code = 0, want non-zero") + } + if !strings.Contains(stderr, "invalid log level") { + t.Fatalf("stderr = %q, want invalid log level error", stderr) + } +} + +func runCLI(t *testing.T, args []string, source fleet.Source) (stdout, stderr string, exitCode int) { + t.Helper() + + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("SITH_LOG_LEVEL", "") + t.Setenv("SITH_LOG_FORMAT", "") + t.Setenv("SITH_KUBECONFIG", "") + + var stdoutBuffer bytes.Buffer + var stderrBuffer bytes.Buffer + exitCode = execute(args, source, &stdoutBuffer, &stderrBuffer) + return stdoutBuffer.String(), stderrBuffer.String(), exitCode +} diff --git a/internal/cli/clusters.go b/internal/cli/clusters.go new file mode 100644 index 0000000..9e7e194 --- /dev/null +++ b/internal/cli/clusters.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func newClustersCommand(options *rootOptions, source fleet.Source) *cobra.Command { + return &cobra.Command{ + Use: "clusters", + Short: "List clusters from the configured fleet source", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if source == nil { + return fmt.Errorf("fleet source is not configured") + } + + result, err := source.Fleet(command.Context()) + if err != nil { + return fmt.Errorf("read fleet from %s source: %w", source.Kind(), err) + } + if result.Clusters == nil { + result.Clusters = []fleet.Cluster{} + } + + if options.output == "json" { + if err := json.NewEncoder(command.OutOrStdout()).Encode(result); err != nil { + return fmt.Errorf("write clusters JSON: %w", err) + } + return nil + } + + if len(result.Clusters) == 0 { + message := fmt.Sprintf("No clusters found (source: %s).", source.Kind()) + if source.Kind() == "stub" { + message = "No clusters found (source: stub — F2.1/#38 not yet implemented)." + } + if _, err := fmt.Fprintln(command.OutOrStdout(), message); err != nil { + return fmt.Errorf("write empty clusters result: %w", err) + } + return nil + } + + return writeClusterTable(command.OutOrStdout(), result.Clusters) + }, + } +} + +func writeClusterTable(output io.Writer, clusters []fleet.Cluster) error { + var rendered bytes.Buffer + table := tabwriter.NewWriter(&rendered, 0, 4, 2, ' ', 0) + if _, err := fmt.Fprintln(table, "NAME\tCONTEXT\tSOURCE\tREACHABLE\tOBSERVED"); err != nil { + return fmt.Errorf("write cluster table header: %w", err) + } + + for _, cluster := range clusters { + contextName := cluster.Context + if contextName == "" { + contextName = "-" + } + observed := "-" + if !cluster.ObservedAt.IsZero() { + observed = cluster.ObservedAt.UTC().Format(time.RFC3339) + } + if _, err := fmt.Fprintf( + table, + "%s\t%s\t%s\t%t\t%s\n", + cluster.Name, + contextName, + cluster.SourceKind, + cluster.Reachable, + observed, + ); err != nil { + return fmt.Errorf("write cluster table row: %w", err) + } + } + + if err := table.Flush(); err != nil { + return fmt.Errorf("flush cluster table: %w", err) + } + if _, err := io.Copy(output, &rendered); err != nil { + return fmt.Errorf("write cluster table: %w", err) + } + + return nil +} diff --git a/internal/cli/hub.go b/internal/cli/hub.go new file mode 100644 index 0000000..eb0c2fa --- /dev/null +++ b/internal/cli/hub.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func newHubCommand() *cobra.Command { + return &cobra.Command{ + Use: "hub", + Short: "Start the governed fleet hub", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if _, err := fmt.Fprintln(command.OutOrStdout(), "sith hub: not yet implemented — hub mode is phase-1+ (E1–E10)."); err != nil { + return fmt.Errorf("write hub status: %w", err) + } + return nil + }, + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 0000000..bea3098 --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package cli wires the Sith command tree to source-abstract domain packages. +package cli + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + + "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/config" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/logging" +) + +type runtimeKey struct{} + +type runtimeState struct { + config config.Config + logger *slog.Logger +} + +type rootOptions struct { + configPath string + logLevel string + logFormat string + output string +} + +// Execute builds and runs the command tree, returning a process exit code. +func Execute() int { + return execute(os.Args[1:], fleet.StubSource{}, os.Stdout, os.Stderr) +} + +func execute(args []string, source fleet.Source, stdout, stderr io.Writer) int { + command := newRootCommand(source, stdout, stderr) + command.SetArgs(args) + if err := command.Execute(); err != nil { + if _, writeErr := fmt.Fprintln(stderr, err); writeErr != nil { + return 2 + } + return 1 + } + + return 0 +} + +func newRootCommand(source fleet.Source, stdout, stderr io.Writer) *cobra.Command { + options := &rootOptions{output: "text"} + command := &cobra.Command{ + Use: "sith", + Short: "ArdurAI's local-first Kubernetes fleet client", + Long: "Sith is ArdurAI's local-first client for source-abstract, cross-cluster Kubernetes fleet operations.", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(command *cobra.Command, _ []string) error { + return command.Help() + }, + PersistentPreRunE: func(command *cobra.Command, _ []string) error { + if options.output != "text" && options.output != "json" { + return fmt.Errorf("invalid output format %q: expected text or json", options.output) + } + + resolved, err := config.Load(options.configPath, config.Overrides{ + LogLevel: options.logLevel, + LogFormat: options.logFormat, + }) + if err != nil { + return fmt.Errorf("load configuration: %w", err) + } + + logger, err := logging.New(stderr, resolved.LogLevel, resolved.LogFormat) + if err != nil { + return fmt.Errorf("configure logging: %w", err) + } + + state := runtimeState{config: resolved, logger: logger} + command.SetContext(context.WithValue(command.Context(), runtimeKey{}, state)) + return nil + }, + } + command.SetOut(stdout) + command.SetErr(stderr) + command.CompletionOptions.DisableDefaultCmd = true + + flags := command.PersistentFlags() + flags.StringVar(&options.configPath, "config", "", "path to the YAML configuration file") + flags.StringVar(&options.logLevel, "log-level", "", "logging level: debug, info, warn, or error (default info)") + flags.StringVar(&options.logFormat, "log-format", "", "logging format: text or json (default text)") + flags.StringVarP(&options.output, "output", "o", "text", "output format: text or json") + + command.AddCommand( + newVersionCommand(options), + newClustersCommand(options, source), + newUICommand(), + newHubCommand(), + ) + + return command +} diff --git a/internal/cli/ui.go b/internal/cli/ui.go new file mode 100644 index 0000000..6cecd34 --- /dev/null +++ b/internal/cli/ui.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func newUICommand() *cobra.Command { + return &cobra.Command{ + Use: "ui", + Short: "Start the local fleet IDE", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if _, err := fmt.Fprintln(command.OutOrStdout(), "sith ui: not yet implemented — see F11.3 (#34)."); err != nil { + return fmt.Errorf("write ui status: %w", err) + } + return nil + }, + } +} diff --git a/internal/cli/version.go b/internal/cli/version.go new file mode 100644 index 0000000..cc0a51f --- /dev/null +++ b/internal/cli/version.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/buildinfo" +) + +func newVersionCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print build information", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + info := buildinfo.Get() + if options.output == "json" { + encoded, err := info.JSON() + if err != nil { + return err + } + if _, err := fmt.Fprintln(command.OutOrStdout(), encoded); err != nil { + return fmt.Errorf("write version output: %w", err) + } + return nil + } + + if _, err := fmt.Fprintln(command.OutOrStdout(), info.String()); err != nil { + return fmt.Errorf("write version output: %w", err) + } + return nil + }, + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..717cbe0 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package config loads and validates local Sith configuration. +package config + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "go.yaml.in/yaml/v3" +) + +const maxConfigBytes = 1 << 20 + +// Config contains local-mode process settings. +type Config struct { + LogLevel string `yaml:"log_level"` + LogFormat string `yaml:"log_format"` + KubeconfigPath string `yaml:"kubeconfig_path"` +} + +// Overrides contains non-empty command-line values that take final precedence. +type Overrides struct { + LogLevel string + LogFormat string +} + +// Defaults returns the safe local-mode defaults. +func Defaults() Config { + return Config{ + LogLevel: "info", + LogFormat: "text", + } +} + +// Load resolves defaults, an optional YAML file, environment variables, and flag overrides. +func Load(path string, overrides Overrides) (Config, error) { + resolvedPath, explicit, err := resolvePath(path) + if err != nil { + return Config{}, err + } + + resolved := Defaults() + if err := mergeFile(resolvedPath, explicit, &resolved); err != nil { + return Config{}, err + } + + applyEnvironment(&resolved) + applyOverrides(overrides, &resolved) + + if err := resolved.Validate(); err != nil { + return Config{}, err + } + + return resolved, nil +} + +// Validate rejects unknown logging values rather than silently weakening behavior. +func (c Config) Validate() error { + switch c.LogLevel { + case "debug", "info", "warn", "error": + default: + return fmt.Errorf("invalid log level %q: expected debug, info, warn, or error", c.LogLevel) + } + + switch c.LogFormat { + case "text", "json": + default: + return fmt.Errorf("invalid log format %q: expected text or json", c.LogFormat) + } + + return nil +} + +func resolvePath(path string) (resolved string, explicit bool, err error) { + if path != "" { + return path, true, nil + } + + if root := os.Getenv("XDG_CONFIG_HOME"); root != "" { + return filepath.Join(root, "sith", "config.yaml"), false, nil + } + + home, err := os.UserHomeDir() + if err != nil { + return "", false, fmt.Errorf("resolve home directory: %w", err) + } + + return filepath.Join(home, ".config", "sith", "config.yaml"), false, nil +} + +func mergeFile(path string, explicit bool, resolved *Config) error { + data, err := readConfig(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) && !explicit { + return nil + } + + return fmt.Errorf("read config %q: %w", path, err) + } + + if len(bytes.TrimSpace(data)) == 0 { + return nil + } + + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(resolved); err != nil { + return fmt.Errorf("decode config %q: %w", path, err) + } + + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + if err == nil { + return fmt.Errorf("decode config %q: multiple YAML documents are not supported", path) + } + + return fmt.Errorf("decode config %q: %w", path, err) + } + + return nil +} + +func readConfig(path string) ([]byte, error) { + // The path is explicitly selected by the local user or resolved under their config directory. + file, err := os.Open(path) //nolint:gosec // reading that user-selected path is the intended behavior + if err != nil { + return nil, err + } + defer func() { + _ = file.Close() + }() + + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat: %w", err) + } + if info.Size() > maxConfigBytes { + return nil, fmt.Errorf("file is %d bytes, maximum is %d", info.Size(), maxConfigBytes) + } + + data, err := io.ReadAll(io.LimitReader(file, maxConfigBytes+1)) + if err != nil { + return nil, fmt.Errorf("read: %w", err) + } + if len(data) > maxConfigBytes { + return nil, fmt.Errorf("file exceeds maximum size of %d bytes", maxConfigBytes) + } + + return data, nil +} + +func applyEnvironment(resolved *Config) { + if value := os.Getenv("SITH_LOG_LEVEL"); value != "" { + resolved.LogLevel = value + } + if value := os.Getenv("SITH_LOG_FORMAT"); value != "" { + resolved.LogFormat = value + } + if value := os.Getenv("SITH_KUBECONFIG"); value != "" { + resolved.KubeconfigPath = value + } +} + +func applyOverrides(overrides Overrides, resolved *Config) { + if overrides.LogLevel != "" { + resolved.LogLevel = overrides.LogLevel + } + if overrides.LogFormat != "" { + resolved.LogFormat = overrides.LogFormat + } +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..3856fa3 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDefaults(t *testing.T) { + t.Parallel() + + want := Config{LogLevel: "info", LogFormat: "text"} + if got := Defaults(); got != want { + t.Fatalf("Defaults() = %#v, want %#v", got, want) + } +} + +func TestLoadFromFile(t *testing.T) { + clearConfigEnvironment(t) + path := writeConfig(t, "log_level: debug\nlog_format: json\n") + + got, err := Load(path, Overrides{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if got.LogLevel != "debug" || got.LogFormat != "json" { + t.Fatalf("Load() = %#v, want file values", got) + } +} + +func TestEnvOverridesFile(t *testing.T) { + clearConfigEnvironment(t) + path := writeConfig(t, "log_level: debug\nlog_format: json\n") + t.Setenv("SITH_LOG_LEVEL", "warn") + t.Setenv("SITH_LOG_FORMAT", "text") + + got, err := Load(path, Overrides{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if got.LogLevel != "warn" || got.LogFormat != "text" { + t.Fatalf("Load() = %#v, want environment values", got) + } +} + +func TestOverridesBeatEnv(t *testing.T) { + clearConfigEnvironment(t) + t.Setenv("SITH_LOG_LEVEL", "warn") + t.Setenv("SITH_LOG_FORMAT", "json") + + got, err := Load("", Overrides{LogLevel: "error", LogFormat: "text"}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if got.LogLevel != "error" || got.LogFormat != "text" { + t.Fatalf("Load() = %#v, want flag overrides", got) + } +} + +func TestMissingDefaultFileIsOK(t *testing.T) { + clearConfigEnvironment(t) + + got, err := Load("", Overrides{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if got != Defaults() { + t.Fatalf("Load() = %#v, want %#v", got, Defaults()) + } +} + +func TestExplicitMissingPathErrors(t *testing.T) { + clearConfigEnvironment(t) + + _, err := Load(filepath.Join(t.TempDir(), "missing.yaml"), Overrides{}) + if err == nil { + t.Fatal("Load() error = nil, want an error") + } +} + +func TestInvalidLevelRejected(t *testing.T) { + t.Parallel() + + for _, level := range []string{"", "verbose", "INFO"} { + level := level + t.Run(level, func(t *testing.T) { + t.Parallel() + if err := (Config{LogLevel: level, LogFormat: "text"}).Validate(); err == nil { + t.Fatalf("Validate() error = nil for log level %q", level) + } + }) + } +} + +func TestInvalidFormatRejected(t *testing.T) { + t.Parallel() + + for _, format := range []string{"", "console", "JSON"} { + format := format + t.Run(format, func(t *testing.T) { + t.Parallel() + if err := (Config{LogLevel: "info", LogFormat: format}).Validate(); err == nil { + t.Fatalf("Validate() error = nil for log format %q", format) + } + }) + } +} + +func TestUnknownFieldRejected(t *testing.T) { + clearConfigEnvironment(t) + path := writeConfig(t, "log_level: info\ntelemetry: true\n") + + if _, err := Load(path, Overrides{}); err == nil { + t.Fatal("Load() error = nil, want unknown field rejection") + } +} + +func TestMultipleDocumentsRejected(t *testing.T) { + clearConfigEnvironment(t) + path := writeConfig(t, "log_level: info\n---\nlog_format: json\n") + + if _, err := Load(path, Overrides{}); err == nil { + t.Fatal("Load() error = nil, want multiple document rejection") + } +} + +func TestOversizedFileRejected(t *testing.T) { + clearConfigEnvironment(t) + path := writeConfig(t, strings.Repeat("x", maxConfigBytes+1)) + + if _, err := Load(path, Overrides{}); err == nil { + t.Fatal("Load() error = nil, want oversized file rejection") + } +} + +func TestKubeconfigEnvironmentApplied(t *testing.T) { + clearConfigEnvironment(t) + t.Setenv("SITH_KUBECONFIG", "/tmp/fleet-kubeconfig") + + got, err := Load("", Overrides{}) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if got.KubeconfigPath != "/tmp/fleet-kubeconfig" { + t.Fatalf("KubeconfigPath = %q", got.KubeconfigPath) + } +} + +func clearConfigEnvironment(t *testing.T) { + t.Helper() + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("SITH_LOG_LEVEL", "") + t.Setenv("SITH_LOG_FORMAT", "") + t.Setenv("SITH_KUBECONFIG", "") +} + +func writeConfig(t *testing.T, contents string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + return path +} diff --git a/internal/fleet/fleet_test.go b/internal/fleet/fleet_test.go new file mode 100644 index 0000000..b5232c8 --- /dev/null +++ b/internal/fleet/fleet_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package fleet + +import ( + "context" + "encoding/json" + "testing" +) + +var _ Source = StubSource{} +var _ Source = memorySource{} + +type memorySource struct { + result FleetResult +} + +func (memorySource) Kind() string { + return "memory" +} + +func (source memorySource) Fleet(_ context.Context) (FleetResult, error) { + return source.result, nil +} + +func TestStubSourceKind(t *testing.T) { + t.Parallel() + + if got := (StubSource{}).Kind(); got != "stub" { + t.Fatalf("Kind() = %q, want stub", got) + } +} + +func TestStubSourceEmpty(t *testing.T) { + t.Parallel() + + got, err := (StubSource{}).Fleet(context.Background()) + if err != nil { + t.Fatalf("Fleet() error = %v", err) + } + if got.Clusters == nil || len(got.Clusters) != 0 { + t.Fatalf("Clusters = %#v, want allocated empty slice", got.Clusters) + } + if got.Coverage.Requested != 0 || got.Coverage.Reachable != 0 || got.Coverage.Unreachable != nil { + t.Fatalf("Coverage = %#v, want zero value", got.Coverage) + } +} + +func TestSourceInterfaceSatisfied(t *testing.T) { + t.Parallel() + + sources := []Source{ + StubSource{}, + memorySource{result: FleetResult{Clusters: []Cluster{{Name: "lab"}}}}, + } + + for _, source := range sources { + if _, err := source.Fleet(context.Background()); err != nil { + t.Fatalf("%s Fleet() error = %v", source.Kind(), err) + } + } +} + +func TestFleetResultJSONShape(t *testing.T) { + t.Parallel() + + result := FleetResult{Clusters: []Cluster{}, Coverage: Coverage{}} + encoded, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal fleet result: %v", err) + } + + const want = `{"clusters":[],"coverage":{"requested":0,"reachable":0}}` + if string(encoded) != want { + t.Fatalf("JSON = %s, want %s", encoded, want) + } +} diff --git a/internal/fleet/model.go b/internal/fleet/model.go new file mode 100644 index 0000000..b1b9552 --- /dev/null +++ b/internal/fleet/model.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package fleet defines the source-abstract fleet snapshot consumed by Sith surfaces. +package fleet + +import "time" + +// FleetResult is the normalized snapshot returned by a Source. +// +//nolint:revive // FleetResult is the locked cross-slice contract name from issue #47. +type FleetResult struct { + Clusters []Cluster `json:"clusters"` + Coverage Coverage `json:"coverage"` +} + +// Cluster is one cluster or context in the fleet, stamped with source and freshness. +type Cluster struct { + Name string `json:"name"` + Context string `json:"context,omitempty"` + SourceKind string `json:"source_kind"` + Reachable bool `json:"reachable"` + ObservedAt time.Time `json:"observed_at,omitempty"` +} + +// Coverage summarizes which requested scopes answered and which were unreachable. +type Coverage struct { + Requested int `json:"requested"` + Reachable int `json:"reachable"` + Unreachable []string `json:"unreachable,omitempty"` +} diff --git a/internal/fleet/source.go b/internal/fleet/source.go new file mode 100644 index 0000000..e150ebd --- /dev/null +++ b/internal/fleet/source.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +package fleet + +import "context" + +// Source is the read seam implemented by every fleet snapshot backend. +type Source interface { + // Kind identifies the adapter, such as stub, local-kubeconfig, or ocm-spoke. + Kind() string + // Fleet returns the current normalized fleet snapshot for this source. + Fleet(ctx context.Context) (FleetResult, error) +} diff --git a/internal/fleet/stub.go b/internal/fleet/stub.go new file mode 100644 index 0000000..d6ccc70 --- /dev/null +++ b/internal/fleet/stub.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package fleet + +import "context" + +var _ Source = StubSource{} + +// StubSource returns a well-formed empty fleet until Slice 1 adds kubeconfig discovery. +type StubSource struct{} + +// Kind identifies this source as the Slice 0 stub. +func (StubSource) Kind() string { + return "stub" +} + +// Fleet returns an allocated empty cluster slice and zero coverage. +func (StubSource) Fleet(_ context.Context) (FleetResult, error) { + return FleetResult{ + Clusters: []Cluster{}, + Coverage: Coverage{}, + }, nil +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 0000000..9ddd1f0 --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package logging constructs structured loggers for Sith commands. +package logging + +import ( + "fmt" + "io" + "log/slog" +) + +// New builds a structured logger at the requested level and output format. +func New(w io.Writer, level, format string) (*slog.Logger, error) { + resolvedLevel, err := parseLevel(level) + if err != nil { + return nil, err + } + + options := &slog.HandlerOptions{Level: resolvedLevel} + var handler slog.Handler + switch format { + case "text": + handler = slog.NewTextHandler(w, options) + case "json": + handler = slog.NewJSONHandler(w, options) + default: + return nil, fmt.Errorf("invalid log format %q: expected text or json", format) + } + + return slog.New(handler), nil +} + +func parseLevel(level string) (slog.Level, error) { + switch level { + case "debug": + return slog.LevelDebug, nil + case "info": + return slog.LevelInfo, nil + case "warn": + return slog.LevelWarn, nil + case "error": + return slog.LevelError, nil + default: + return 0, fmt.Errorf("invalid log level %q: expected debug, info, warn, or error", level) + } +} diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go new file mode 100644 index 0000000..23aef9d --- /dev/null +++ b/internal/logging/logging_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package logging + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestNewTextHandler(t *testing.T) { + t.Parallel() + + var output bytes.Buffer + logger, err := New(&output, "info", "text") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + logger.Info("ready", "clusters", 0) + if got := output.String(); !strings.Contains(got, "level=INFO") || !strings.Contains(got, "msg=ready") { + t.Fatalf("text log = %q, want level and message", got) + } + if strings.HasPrefix(strings.TrimSpace(output.String()), "{") { + t.Fatalf("text log unexpectedly looks like JSON: %q", output.String()) + } +} + +func TestNewJSONHandler(t *testing.T) { + t.Parallel() + + var output bytes.Buffer + logger, err := New(&output, "info", "json") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + logger.Info("ready", "clusters", 0) + var entry map[string]any + if err := json.Unmarshal(output.Bytes(), &entry); err != nil { + t.Fatalf("unmarshal log line %q: %v", output.String(), err) + } + if entry["msg"] != "ready" || entry["level"] != "INFO" { + t.Fatalf("JSON log = %#v, want ready/INFO", entry) + } +} + +func TestLevelFiltering(t *testing.T) { + t.Parallel() + + var output bytes.Buffer + logger, err := New(&output, "warn", "text") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + logger.Info("hidden") + logger.Warn("visible") + if strings.Contains(output.String(), "hidden") || !strings.Contains(output.String(), "visible") { + t.Fatalf("filtered output = %q", output.String()) + } +} + +func TestInvalidLevelOrFormatErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + level string + format string + }{ + {name: "level", level: "verbose", format: "text"}, + {name: "format", level: "info", format: "console"}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if _, err := New(&bytes.Buffer{}, test.level, test.format); err == nil { + t.Fatal("New() error = nil, want an error") + } + }) + } +} diff --git a/sessions/2026-07-10-build-plan-architecture.md b/sessions/2026-07-10-build-plan-architecture.md new file mode 100644 index 0000000..2caa92a --- /dev/null +++ b/sessions/2026-07-10-build-plan-architecture.md @@ -0,0 +1,45 @@ +# Session — 2026-07-10 — build-plan-architecture + +**Builder:** GR (architect/lead role) · **Model/effort:** Opus 4.8, max · **Branch:** docs/build-plan +**Slice(s):** Phase-L planning — locks the slice sequence + Slice-0 spec + conventions · **Status:** done + +--- + +[G] Goal: produce the locked Phase-L build plan for the local fleet wedge — `docs/BUILD-SEQUENCE.md`, +`docs/specs/SLICE-0-foundation.md`, `docs/CONVENTIONS.md` — so a Sonnet builder can start Slice 0 with +no extra context. Grounded in `docs/SITH-NOTION.md` and issues #29/#38/#32/#33/#34/#35/#36/#37/#39. +[S] Scope: markdown docs + the `sessions/` GSTACK scaffold only. No product Go code. Off `dev`, PR into +`dev`, do not touch `main`, do not merge. +[A] Action: read SITH-NOTION.md (E2/E7/E9/E11 epics, roadmap map, open questions Q12–Q15), ADR-0002 +(Go/single-binary), and all Phase-L issues. Locked the slice order (0→F2.1+F11.1→F11.2→F11.5→F11.3→ +F11.6→F7.1, plus parallel packaging track P) and validated it against the doc; recorded the one +divergence from roadmap #39 (E9 packaging folded to a parallel non-gating track). Wrote the three +deliverables + the GSTACK journal scaffold (`sessions/README.md`, `JOURNAL-TEMPLATE.md`). +[T] Test: link-checked relative doc references; confirmed referenced files exist; confirmed signing +config (SSH ED25519, gpgsign on) and branch base (`dev`, tip 6b81428). No code to run this session. +[C] Checkpoint #1: 15def82 — build plan + Slice-0 spec + conventions + sessions scaffold on +`docs/build-plan`; PR #40 into `dev`. next: hand Slice-0 spec to the Sonnet builder. + +[G] Goal: weight the build sequence toward GR's real day-to-day — K8s, Helm, ArgoCD, Docker, +Python/bash, Fluentd/Fluent-bit, Grafana/Prometheus, multi-cloud AWS/Azure/GCP, vuln fixes, cloud +networking — and leave a hook for a fuller GR-workflow profile supplied next. +[S] Scope: `docs/BUILD-SEQUENCE.md` only. Slice 0 stays workflow-agnostic. No spec/convention changes. +[A] Action: added a "Who this is for — the target user's daily surface" section (stack → plan mapping) +with a hook to a forthcoming `docs/GR-WORKFLOW-PROFILE.md`; added a concrete "User-workflow fit" +line to every slice (0 agnostic; 1 morning fleet sweep; 2 incident triage + vuln sweep; 3 debug the +failing pod; 4 GUI/share; 5 run safely on a corp laptop; 6 ask the agent; P install like kubectl). +Redone in a dedicated worktree (`/Volumes/EXTENDED/repos/sith-build-plan`) after a concurrent builder +session force-switched the shared main worktree to `docs/f11-local-fleet-ux` and discarded the +uncommitted first pass. +[T] Test: grep confirms 8 per-slice fit lines + the hook; anchors/links intact. +[C] Checkpoint #2: — user-workflow weighting folded into BUILD-SEQUENCE; next: fold the +fuller GR-workflow profile into `docs/GR-WORKFLOW-PROFILE.md` when GR supplies it, and re-rank E12/E13 +connectors to match. + +--- + +**Session close:** Phase-L plan locked; Slice-0 spec is self-contained and ready for a fresh builder. +**Open questions touched:** Q12 (hero surface → TUI/CLI first, `sith ui` fast-follow); Q13 (local→hub +upgrade → deferred to phase-1+, seam only); Q14 (local MCP auth → loopback + optional keychain token); +Q15 (telemetry → permanent hard no in Phase L). All defaults recorded in `docs/BUILD-SEQUENCE.md`; +owner may override any without disturbing Slice 0. diff --git a/sessions/2026-07-10-slice-0-foundation.md b/sessions/2026-07-10-slice-0-foundation.md new file mode 100644 index 0000000..97d994d --- /dev/null +++ b/sessions/2026-07-10-slice-0-foundation.md @@ -0,0 +1,53 @@ +# Session — 2026-07-10 — slice-0-foundation + +**Builder:** Gnani Rahul · **Model/effort:** engineering, max · **Branch:** feat/slice-0-foundation +**Slice(s):** Slice 0 / #47 · **Status:** in-progress + +--- + +[G] Goal: Land the Slice 0 walking skeleton from issue #47 on `dev`: build metadata, config, +structured logging, the typed `fleet.Source` stub seam, the Cobra CLI, tests, and green CI. +[S] Scope: `cmd/sith`, `internal/{buildinfo,config,logging,fleet,cli}`, build/CI files, binary smoke +tests, README, and the GSTACK scaffold. Kubeconfig/client-go, TUI, web UI, MCP, keychain, and hub +implementation are explicitly out of scope. +[A] Action: Verified the canonical checkout at `/Volumes/EXTENDED/repos/sith`, read the locked specs +and issue #47, created `feat/slice-0-foundation` from `origin/dev`, and isolated the work in +`/Volumes/EXTENDED/repos/sith-slice-0`. +[A] Action: Replaced three stale spec pins with supported equivalents: Go 1.25 (supported +oldstable), the maintained `go.yaml.in/yaml/v3` fork, and current supported GitHub Actions plus +golangci-lint v2.12.2. Product behavior and slice boundaries are unchanged. +[A] Action: Implemented the typed empty `fleet.Source` path, build metadata, fail-safe configuration, +structured logging, deterministic CLI text/JSON rendering, UI/hub stubs, and a process-level binary +smoke suite. Updated the README from planning-only status to the runnable Slice 0 surface. +[T] Test: `make ci` passed with Go 1.25.12 and golangci-lint v2.12.2: gofmt/goimports, `go vet`, +11 strict linters, race-enabled unit tests, coverage, subprocess e2e tests, and the ldflags build. +Core package coverage is 81.1% CLI, 83.8% config, 83.3% buildinfo, 87.5% logging, and 100% fleet. +[T] Test: `govulncheck ./...` reported no vulnerabilities; `go mod verify`, 20 shuffled test +repetitions, action-SHA verification, forbidden-attribution/product-name scans, SPDX checks, and +manual command/exit-code smoke checks passed. The external CodeRabbit CLI was unavailable, so the +review remained local and no repository data was uploaded. +[C] Checkpoint #1: f9ae42d — Go module, dependency, and strict quality-tool baseline; next: core +packages. +[C] Checkpoint #2: 5383365 — buildinfo, config, logging, and typed fleet seam with tests; next: CLI. +[C] Checkpoint #3: 35f1190 — runnable Cobra walking skeleton and binary e2e suite; next: CI. +[C] Checkpoint #4: ab9f59b — least-privilege, SHA-pinned GitHub Actions merge gates; next: session +documentation and PR publication. +[C] Checkpoint #5: 5a488ed — README and GSTACK session record; next: push, PR into `dev`, and +remote CI/review. +[T] Test: PR #50's first CI run failed in the lint action because current action v9 supplies the +`run` subcommand itself; `args: run ./...` became `run run ./...` and treated `run/` as a package. +The product build and local lint remained green. +[A] Action: Corrected the action input to `args: ./...`, matching the current official action +contract while preserving the exact local `golangci-lint run ./...` gate. +[C] Checkpoint #6: dd738d7 — repair the remote lint-action invocation; next: push and re-run CI. +[T] Test: PR #50's repaired CI run 29116977271 passed all remote gates. A requirement audit then +identified that ldflags injection through `make build` had only manual evidence, not an automated +assertion. +[A] Action: Added a subprocess integration test that invokes the real Makefile with deterministic +version, commit, and date values, then parses `sith version -o json` and asserts all three fields. +[C] Checkpoint #7: this commit — automate the Makefile metadata acceptance criterion; next: push, +re-run CI, and await independent review. + +--- + +**Session close:** implementation complete; remote CI/review pending · **Open questions touched:** none diff --git a/sessions/JOURNAL-TEMPLATE.md b/sessions/JOURNAL-TEMPLATE.md new file mode 100644 index 0000000..6ce7462 --- /dev/null +++ b/sessions/JOURNAL-TEMPLATE.md @@ -0,0 +1,18 @@ +# Session — + +**Builder:** · **Model/effort:** · **Branch:** feat/ +**Slice(s):** · **Status:** in-progress | done | blocked + +--- + +[G] Goal: +[S] Scope: +[A] Action: +[T] Test: +[C] Checkpoint #1: ; next: + + + +--- + +**Session close:** · **Open questions touched:** diff --git a/sessions/README.md b/sessions/README.md new file mode 100644 index 0000000..bef7ac0 --- /dev/null +++ b/sessions/README.md @@ -0,0 +1,29 @@ +# Sith session journals (GSTACK) + +This directory is Sith's **session journal**, kept under the GSTACK discipline defined in +[`../docs/CONVENTIONS.md`](../docs/CONVENTIONS.md) §3. + +Every build session (a continuous stretch of work by one builder) copies +[`JOURNAL-TEMPLATE.md`](JOURNAL-TEMPLATE.md) to `YYYY-MM-DD-.md` and appends entries as +it works, so the next session — human or agent — resumes with full context and every checkpoint maps +to a commit. + +## Entry markers + +| Marker | Name | Records | +|---|---|---| +| `[G]` | Goal | The objective of the work unit + issue number(s). | +| `[S]` | Scope | Files/packages in play; what is explicitly out. | +| `[A]` | Action | What was actually done. | +| `[T]` | Test | How it was verified + result. | +| `[C]` | Checkpoint | A numbered milestone: commit SHA(s), decision, next step. `#` matches the commit's `GSTACK-Checkpoint` trailer. | + +The journal is the **stack** of these G/S/A/T/C entries — hence *GSTACK*. + +## Rules + +- Start a session by copying the template; append as you go (do not reconstruct at the end). +- Each `[C]` checkpoint ⇄ exactly one commit carrying `GSTACK-Checkpoint: YYYY-MM-DD/#`. +- Record which open questions (Q12–Q15, `docs/SITH-NOTION.md` §9) a slice touched and the default chosen. +- This directory is committed (engineering history). **Never** put secrets, tokens, kubeconfigs, or + customer data in a journal. diff --git a/tests/e2e/smoke_test.go b/tests/e2e/smoke_test.go new file mode 100644 index 0000000..1f198e7 --- /dev/null +++ b/tests/e2e/smoke_test.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e + +// Package e2e_test verifies the compiled Sith process boundary. +package e2e_test + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestBinarySmoke(t *testing.T) { + root := repositoryRoot(t) + binary := filepath.Join(t.TempDir(), "sith") + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + build := exec.CommandContext(ctx, "go", "build", "-trimpath", "-o", binary, "./cmd/sith") + build.Dir = root + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build binary: %v\n%s", err, output) + } + + tests := []struct { + name string + args []string + contains string + validJSON bool + }{ + {name: "version text", args: []string{"version"}, contains: "sith dev"}, + {name: "version JSON", args: []string{"version", "-o", "json"}, validJSON: true}, + {name: "clusters text", args: []string{"clusters"}, contains: "No clusters found"}, + {name: "clusters JSON", args: []string{"clusters", "-o", "json"}, validJSON: true}, + {name: "ui stub", args: []string{"ui"}, contains: "not yet implemented"}, + {name: "hub stub", args: []string{"hub"}, contains: "phase-1+"}, + {name: "no arguments", contains: "Usage:"}, + {name: "help", args: []string{"--help"}, contains: "Usage:"}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + command := exec.CommandContext(ctx, binary, test.args...) + command.Env = append(os.Environ(), "XDG_CONFIG_HOME="+t.TempDir()) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("run %v: %v\n%s", test.args, err, output) + } + if test.contains != "" && !strings.Contains(string(output), test.contains) { + t.Fatalf("output = %q, want %q", output, test.contains) + } + if test.validJSON && !json.Valid(output) { + t.Fatalf("output is not valid JSON: %q", output) + } + }) + } +} + +func TestUnknownCommandFails(t *testing.T) { + t.Parallel() + + root := repositoryRoot(t) + binary := filepath.Join(t.TempDir(), "sith") + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + build := exec.CommandContext(ctx, "go", "build", "-trimpath", "-o", binary, "./cmd/sith") + build.Dir = root + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build binary: %v\n%s", err, output) + } + + command := exec.CommandContext(ctx, binary, "bogus") + output, err := command.CombinedOutput() + if err == nil { + t.Fatalf("bogus command succeeded: %s", output) + } + if !strings.Contains(string(output), "unknown command") { + t.Fatalf("output = %q, want unknown command", output) + } +} + +func TestMakeBuildInjectsMetadata(t *testing.T) { + root := repositoryRoot(t) + binDir := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + const ( + version = "v0.0.0-e2e" + commit = "abc1234" + date = "2026-07-10T19:00:00Z" + ) + build := exec.CommandContext( + ctx, + "make", + "build", + "BIN_DIR="+binDir, + "VERSION="+version, + "COMMIT="+commit, + "DATE="+date, + ) + build.Dir = root + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("make build: %v\n%s", err, output) + } + + binary := filepath.Join(binDir, "sith") + command := exec.CommandContext(ctx, binary, "version", "-o", "json") + output, err := command.Output() + if err != nil { + t.Fatalf("run built binary: %v", err) + } + + var got struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + } + if err := json.Unmarshal(output, &got); err != nil { + t.Fatalf("unmarshal version output %q: %v", output, err) + } + if got.Version != version || got.Commit != commit || got.Date != date { + t.Fatalf("metadata = %#v, want version=%q commit=%q date=%q", got, version, commit, date) + } +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve current test file") + } + + return filepath.Clean(filepath.Join(filepath.Dir(currentFile), "..", "..")) +}