ci: scaffold e2e testing framework infrastructure

.github/workflows/ws-e2e-test.yml

@@ -0,0 +1,53 @@
+name: Workspaces E2E Tests
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main
+      - notebooks-v2
+      - v*-branch
+  pull_request:
+    paths:
+      - 'workspaces/**'
+      - 'releasing/version/VERSION'
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    env:
+      GO_VERSION: '1.22'
+    defaults:
+      run:
+        working-directory: testing
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+
+      - name: Generate cache key from Makefile
+        id: cache-key
+        run: |
+          VERSION_HASH=$(make dependency-hash)
+          echo "cache_key=testing-bin-${{ runner.os }}-go${{ env.GO_VERSION }}-$VERSION_HASH" >> $GITHUB_OUTPUT
+
+      - name: Cache testing/bin directory
+        uses: actions/cache@v4
+        id: cache-testing-bin
+        with:
+          path: testing/bin
+          key: ${{ steps.cache-key.outputs.cache_key }}
+
+      - name: Setup cluster
+        run: make setup-cluster
+
+      - name: Run local-e2e tests
+        run: make local-e2e
+

testing/.gitignore

@@ -0,0 +1,2 @@
+# Binaries for programs and plugins
+bin/*

testing/Makefile

@@ -0,0 +1,161 @@
+GIT_COMMIT     := $(shell git rev-parse HEAD)
+GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
+
+# Image URL to use all building/pushing image targets
+REGISTRY ?= ghcr.io/kubeflow/notebooks
+TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
+
+CONTROLLER_NAME ?= workspaces-controller
+CONTROLLER_IMG ?= $(REGISTRY)/$(CONTROLLER_NAME):$(TAG)
+
+BACKEND_NAME ?= workspaces-backend
+BACKEND_IMG ?= $(REGISTRY)/$(BACKEND_NAME):$(TAG)
+
+FRONTEND_NAME ?= workspaces-frontend
+FRONTEND_IMG ?= $(REGISTRY)/$(FRONTEND_NAME):$(TAG)
+
+KIND_CLUSTER_NAME ?= local-e2e
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+
+# Export KIND_EXPERIMENTAL_PROVIDER to honor it if set in user's environment
+# (e.g., KIND_EXPERIMENTAL_PROVIDER=podman for podman support)
+export KIND_EXPERIMENTAL_PROVIDER
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk command is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Deployment
+
+.PHONY: build-controller build-backend build-frontend build-all e2e
+
+deploy-controller: kind check-kind-context ## Build and deploy the controller.
+	cd ../workspaces/controller && $(MAKE) docker-build IMG=$(CONTROLLER_IMG)
+	$(KIND) load docker-image $(CONTROLLER_IMG) --name $(KIND_CLUSTER_NAME)
+	cd ../workspaces/controller && $(MAKE) deploy IMG=$(CONTROLLER_IMG)
+
+deploy-backend: kind check-kind-context ## Build and deploy the backend.
+	cd ../workspaces/backend && $(MAKE) docker-build IMG=$(BACKEND_IMG)
+	$(KIND) load docker-image $(BACKEND_IMG) --name $(KIND_CLUSTER_NAME)
+	cd ../workspaces/backend && $(MAKE) deploy IMG=$(BACKEND_IMG)
+
+deploy-frontend: kind check-kind-context ## Build and deploy the frontend.
+	cd ../workspaces/frontend && $(MAKE) docker-build IMG=$(FRONTEND_IMG)
+	$(KIND) load docker-image $(FRONTEND_IMG) --name $(KIND_CLUSTER_NAME)
+	cd ../workspaces/frontend && $(MAKE) deploy IMG=$(FRONTEND_IMG)
+
+deploy-all: deploy-controller deploy-backend deploy-frontend ## Deploy all components.
+
+local-e2e: deploy-all istioctl ## Run e2e tests.
+	@echo "TODO: Run e2e tests..."
+
+
+##@ Dependencies
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KIND ?= $(LOCALBIN)/kind
+KUBECTL ?= kubectl
+ISTIOCTL ?= $(LOCALBIN)/istioctl
+
+## Tool Versions
+KIND_VERSION ?= v0.30.0
+ISTIOCTL_VERSION ?= 1.27.3
+
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(LOCALBIN)
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION))
+
+.PHONY: check-kubectl
+check-kubectl: ## Verify that kubectl is available in PATH.
+	@if ! command -v $(KUBECTL) >/dev/null 2>&1; then \
+		echo "✗ ERROR: kubectl is not installed or not found in PATH"; \
+		echo "  Please install kubectl: https://kubernetes.io/docs/tasks/tools/#kubectl"; \
+		exit 1; \
+	fi
+	@echo "✓ kubectl found: $$($(KUBECTL) version --client 2>/dev/null || echo 'version check failed')"
+
+.PHONY: istioctl
+istioctl: $(ISTIOCTL) ## Download istioctl locally if necessary.
+$(ISTIOCTL): $(LOCALBIN)
+	$(call go-install-tool,$(ISTIOCTL),istio.io/istio/istioctl/cmd/istioctl,$(ISTIOCTL_VERSION))
+
+.PHONY: dependency-hash
+dependency-hash: ## Calculate hash of dependency versions for caching.
+	@echo -e "KIND_VERSION=$(KIND_VERSION)\nISTIOCTL_VERSION=$(ISTIOCTL_VERSION)" | sha256sum | cut -d' ' -f1 | head -c 16
+
+.PHONY: setup-cluster
+setup-cluster: check-kubectl kind istioctl ## Set up a complete kind cluster with cert-manager and Istio.
+	@export PATH="$(LOCALBIN):$$PATH" && \
+		bash scripts/setup-kind.sh && \
+		bash scripts/setup-cert-manager.sh && \
+		bash scripts/setup-istio.sh
+	@echo "✓ Cluster setup complete"
+
+.PHONY: check-kind-context
+check-kind-context: check-kubectl ## Verify that the current kubectl context is a kind cluster.
+	@current_context=$$($(KUBECTL) config current-context 2>/dev/null) || { \
+		echo "Error: Unable to get current kubectl context. Is kubectl configured?"; \
+		exit 1; \
+	}; \
+	server_url=$$($(KUBECTL) config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null) || { \
+		echo "Error: Unable to get cluster server URL for context '$$current_context'"; \
+		exit 1; \
+	}; \
+	context_check=0; \
+	server_check=0; \
+	if echo "$$current_context" | grep -qE '^kind-'; then \
+		context_check=1; \
+	fi; \
+	if echo "$$server_url" | grep -qE '(127\.0\.0\.1|localhost)'; then \
+		server_check=1; \
+	fi; \
+	if [ $$context_check -ne 1 ] || [ $$server_check -ne 1 ]; then \
+		echo "✗ ERROR: Current context '$$current_context' does not appear to be a kind cluster!"; \
+		if [ $$context_check -ne 1 ]; then \
+			echo "  ✗ Context name does not match kind-* pattern (got: $$current_context)"; \
+		fi; \
+		if [ $$server_check -ne 1 ]; then \
+			echo "  ✗ Server URL does not use localhost/127.0.0.1 (got: $$server_url)"; \
+		fi; \
+		exit 1; \
+	fi
+
+# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
+# $1 - target path with name of binary
+# $2 - package url which can be installed
+# $3 - specific version of package
+define go-install-tool
+@[ -f "$(1)-$(3)" ] || { \
+set -e; \
+package=$(2)@$(3) ;\
+echo "Downloading $${package}" ;\
+rm -f $(1) || true ;\
+GOBIN=$(LOCALBIN) go install $${package} ;\
+mv $(1) $(1)-$(3) ;\
+} ;\
+ln -sf $(1)-$(3) $(1)
+endef

testing/OWNERS

@@ -0,0 +1,5 @@
+labels:
+  - area/ci
+  - area/v2
+approvers:
+  - andyatmiami

testing/scripts/kind.yml

@@ -0,0 +1,18 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+# This is needed in order to support projected volumes with service account tokens.
+kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta3
+    kind: ClusterConfiguration
+    metadata:
+      name: config
+    apiServer:
+      extraArgs:
+        "service-account-issuer": "kubernetes.default.svc"
+        "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+nodes:
+- role: control-plane
+  image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f
+- role: worker
+  image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f

testing/scripts/setup-cert-manager.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Setup script for cert-manager
+# This script checks if cert-manager is installed and installs it if needed
+# Uses the same version as the e2e tests: v1.12.13 (LTS version)
+
+set -euo pipefail
+
+# Use LTS version of cert-manager (matches e2e tests)
+CERT_MANAGER_VERSION="v1.12.13"
+CERT_MANAGER_URL="https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"
+
+# Check if cert-manager is already installed
+if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
+  echo "Cert-manager is already installed"
+  exit 0
+fi
+
+echo "Installing cert-manager ${CERT_MANAGER_VERSION}..."
+kubectl apply -f "${CERT_MANAGER_URL}"
+
+echo "Waiting for cert-manager to be ready..."
+# Wait for cert-manager webhook to be ready (this is the critical component)
+kubectl wait --for=condition=ready pod \
+  -l app.kubernetes.io/instance=cert-manager \
+  -n cert-manager \
+  --timeout=120s || {
+  echo "Warning: cert-manager pods may not be fully ready, but continuing..."
+}
+
+# Also wait for the CRDs to be established
+kubectl wait --for=condition=established crd/certificates.cert-manager.io --timeout=60s || true
+kubectl wait --for=condition=established crd/issuers.cert-manager.io --timeout=60s || true
+kubectl wait --for=condition=established crd/clusterissuers.cert-manager.io --timeout=60s || true
+
+echo "Cert-manager installation complete"

testing/scripts/setup-istio.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Setup script for Istio
+# This script checks if Istio is installed and installs it if needed
+# Uses istioctl to install the default profile
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TESTING_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+LOCALBIN="${TESTING_DIR}/bin"
+
+# Determine istioctl path - prefer LOCALBIN, fallback to PATH
+if [ -f "${LOCALBIN}/istioctl" ]; then
+  ISTIOCTL="${LOCALBIN}/istioctl"
+elif command -v istioctl >/dev/null 2>&1; then
+  ISTIOCTL="istioctl"
+else
+  echo "ERROR: istioctl is not installed. Please install istioctl first:"
+  echo "  cd testing && make istioctl"
+  echo "  or visit: https://istio.io/latest/docs/setup/getting-started/#download"
+  exit 1
+fi
+
+# Check if Istio is already installed
+# Check for istio-system namespace or istio CRDs
+if kubectl get namespace istio-system >/dev/null 2>&1 && \
+   kubectl get crd virtualservices.networking.istio.io >/dev/null 2>&1; then
+  echo "Istio is already installed"
+  exit 0
+fi
+
+echo "Installing Istio with default profile..."
+"${ISTIOCTL}" install --set profile=default -y
+
+echo "Waiting for Istio to be ready..."
+# Wait for istiod to be ready
+kubectl wait --for=condition=ready pod \
+  -l app=istiod \
+  -n istio-system \
+  --timeout=120s || {
+  echo "Warning: Istio pods may not be fully ready, but continuing..."
+}
+
+# Wait for istio ingress gateway to be ready (if present)
+kubectl wait --for=condition=ready pod \
+  -l app=istio-ingressgateway \
+  -n istio-system \
+  --timeout=120s || {
+  echo "Warning: Istio ingress gateway may not be ready, but continuing..."
+}
+
+echo "Istio installation complete"
+

testing/scripts/setup-kind.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Setup script for Kind cluster
+# This script checks if a Kind cluster exists and creates it if needed
+
+set -euo pipefail
+
+CLUSTER_NAME="local-e2e"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KIND_CONFIG="${SCRIPT_DIR}/kind.yml"
+
+# Check if kind command exists
+if ! command -v kind >/dev/null 2>&1; then
+  echo "ERROR: kind is not installed. Please install kind first:"
+  echo "  brew install kind  # macOS"
+  echo "  or visit: https://kind.sigs.k8s.io/docs/user/quick-start/#installation"
+  exit 1
+fi
+
+# Check if cluster exists
+if ! kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then
+  echo "Creating Kind cluster '${CLUSTER_NAME}' with config from ${KIND_CONFIG}..."
+  kind create cluster --name "${CLUSTER_NAME}" --config "${KIND_CONFIG}" --wait 60s
+  echo "Kind cluster created successfully"
+else
+  echo "Kind cluster '${CLUSTER_NAME}' already exists"
+fi
+
+# Ensure kubectl context is set to the Kind cluster
+kubectl config use-context "kind-${CLUSTER_NAME}" || true
+
+echo "Kind cluster setup complete"

workspaces/backend/.gitignore

@@ -13,5 +13,8 @@ Dockerfile.cross
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Intermediate output from kustomize
+**/kustomize/.output/
+
 # Go workspace file
 go.work

workspaces/backend/Makefile

@@ -145,8 +145,12 @@ $(GOLANGCI_LINT): $(LOCALBIN)
 
 .PHONY: deploy
 deploy: kustomize ## Deploy backend to the K8s cluster specified in ~/.kube/config.
-	cd manifests/kustomize/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG}
-	$(KUBECTL) apply -k manifests/kustomize/overlays/istio
+	@echo "Copying kustomize directory structure to .output..."
+	@rm -rf manifests/kustomize/.output
+	@mkdir -p manifests/kustomize/.output
+	@cp -r manifests/kustomize/* manifests/kustomize/.output/
+	@cd manifests/kustomize/.output/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG}
+	@$(KUBECTL) apply -k manifests/kustomize/.output/overlays/istio
 
 .PHONY: undeploy
 undeploy: kustomize ## Undeploy backend from the K8s cluster specified in ~/.kube/config.