From 0f1461d27b466ee9fba068b8e023a8b097845b1e Mon Sep 17 00:00:00 2001 From: Andy Stoneberg Date: Sat, 22 Nov 2025 14:37:59 -0500 Subject: [PATCH] ci: scaffold e2e testing framework infrastructure This commit establishes the foundational infrastructure for end-to-end testing of the workspaces components. This is an intermediate step that sets up the testing framework; no actual test execution is implemented yet and will be added in subsequent work. Changes include: - Add new `testing/` directory with Makefile and setup scripts: * `setup-kind.sh`: Automated Kind cluster creation and configuration * `setup-cert-manager.sh`: Cert-manager installation (v1.12.13 LTS) * `setup-istio.sh`: Istio service mesh installation * Makefile targets for cluster setup, component deployment, and e2e test execution (currently placeholder with TODO) - Add GitHub Actions workflow (`.github/workflows/ws-e2e-test.yml`): * Triggers on pushes to main branches and PRs affecting workspaces * Sets up Go environment and Kind cluster * Executes local-e2e target (placeholder for future test implementation) - Update workspace Makefiles (backend, controller, frontend): * Modify deploy targets to use `.output/` directories for kustomize operations, preventing modification of source manifests - Update .gitignore files: * Add `.output/` directories to prevent committing generated kustomize artifacts The framework enables automated deployment of all three components (controller, backend, frontend) to a Kind cluster with cert-manager and Istio pre-configured, providing the foundation for comprehensive e2e test scenarios in future commits. Signed-off-by: Andy Stoneberg --- .github/workflows/ws-e2e-test.yml | 53 +++++++++ testing/.gitignore | 2 + testing/Makefile | 161 ++++++++++++++++++++++++++ testing/OWNERS | 5 + testing/scripts/kind.yml | 18 +++ testing/scripts/setup-cert-manager.sh | 36 ++++++ testing/scripts/setup-istio.sh | 54 +++++++++ testing/scripts/setup-kind.sh | 32 +++++ workspaces/backend/.gitignore | 3 + workspaces/backend/Makefile | 8 +- workspaces/controller/.gitignore | 3 + workspaces/controller/Makefile | 8 +- workspaces/frontend/.gitignore | 3 + workspaces/frontend/Makefile | 8 +- 14 files changed, 388 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ws-e2e-test.yml create mode 100644 testing/.gitignore create mode 100644 testing/Makefile create mode 100644 testing/OWNERS create mode 100644 testing/scripts/kind.yml create mode 100755 testing/scripts/setup-cert-manager.sh create mode 100755 testing/scripts/setup-istio.sh create mode 100755 testing/scripts/setup-kind.sh diff --git a/.github/workflows/ws-e2e-test.yml b/.github/workflows/ws-e2e-test.yml new file mode 100644 index 00000000..2b29e767 --- /dev/null +++ b/.github/workflows/ws-e2e-test.yml @@ -0,0 +1,53 @@ +name: Workspaces E2E Tests + +permissions: + contents: read + +on: + push: + branches: + - main + - notebooks-v2 + - v*-branch + pull_request: + paths: + - 'workspaces/**' + - 'releasing/version/VERSION' + +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + GO_VERSION: '1.22' + defaults: + run: + working-directory: testing + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Generate cache key from Makefile + id: cache-key + run: | + VERSION_HASH=$(make dependency-hash) + echo "cache_key=testing-bin-${{ runner.os }}-go${{ env.GO_VERSION }}-$VERSION_HASH" >> $GITHUB_OUTPUT + + - name: Cache testing/bin directory + uses: actions/cache@v4 + id: cache-testing-bin + with: + path: testing/bin + key: ${{ steps.cache-key.outputs.cache_key }} + + - name: Setup cluster + run: make setup-cluster + + - name: Run local-e2e tests + run: make local-e2e + diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 00000000..18dac3ce --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,2 @@ +# Binaries for programs and plugins +bin/* \ No newline at end of file diff --git a/testing/Makefile b/testing/Makefile new file mode 100644 index 00000000..2a10a66a --- /dev/null +++ b/testing/Makefile @@ -0,0 +1,161 @@ +GIT_COMMIT := $(shell git rev-parse HEAD) +GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "") + +# Image URL to use all building/pushing image targets +REGISTRY ?= ghcr.io/kubeflow/notebooks +TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE) + +CONTROLLER_NAME ?= workspaces-controller +CONTROLLER_IMG ?= $(REGISTRY)/$(CONTROLLER_NAME):$(TAG) + +BACKEND_NAME ?= workspaces-backend +BACKEND_IMG ?= $(REGISTRY)/$(BACKEND_NAME):$(TAG) + +FRONTEND_NAME ?= workspaces-frontend +FRONTEND_IMG ?= $(REGISTRY)/$(FRONTEND_NAME):$(TAG) + +KIND_CLUSTER_NAME ?= local-e2e + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + + +# Export KIND_EXPERIMENTAL_PROVIDER to honor it if set in user's environment +# (e.g., KIND_EXPERIMENTAL_PROVIDER=podman for podman support) +export KIND_EXPERIMENTAL_PROVIDER + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Deployment + +.PHONY: build-controller build-backend build-frontend build-all e2e + +deploy-controller: kind check-kind-context ## Build and deploy the controller. + cd ../workspaces/controller && $(MAKE) docker-build IMG=$(CONTROLLER_IMG) + $(KIND) load docker-image $(CONTROLLER_IMG) --name $(KIND_CLUSTER_NAME) + cd ../workspaces/controller && $(MAKE) deploy IMG=$(CONTROLLER_IMG) + +deploy-backend: kind check-kind-context ## Build and deploy the backend. + cd ../workspaces/backend && $(MAKE) docker-build IMG=$(BACKEND_IMG) + $(KIND) load docker-image $(BACKEND_IMG) --name $(KIND_CLUSTER_NAME) + cd ../workspaces/backend && $(MAKE) deploy IMG=$(BACKEND_IMG) + +deploy-frontend: kind check-kind-context ## Build and deploy the frontend. + cd ../workspaces/frontend && $(MAKE) docker-build IMG=$(FRONTEND_IMG) + $(KIND) load docker-image $(FRONTEND_IMG) --name $(KIND_CLUSTER_NAME) + cd ../workspaces/frontend && $(MAKE) deploy IMG=$(FRONTEND_IMG) + +deploy-all: deploy-controller deploy-backend deploy-frontend ## Deploy all components. + +local-e2e: deploy-all istioctl ## Run e2e tests. + @echo "TODO: Run e2e tests..." + + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KIND ?= $(LOCALBIN)/kind +KUBECTL ?= kubectl +ISTIOCTL ?= $(LOCALBIN)/istioctl + +## Tool Versions +KIND_VERSION ?= v0.30.0 +ISTIOCTL_VERSION ?= 1.27.3 + +.PHONY: kind +kind: $(KIND) ## Download kind locally if necessary. +$(KIND): $(LOCALBIN) + $(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION)) + +.PHONY: check-kubectl +check-kubectl: ## Verify that kubectl is available in PATH. + @if ! command -v $(KUBECTL) >/dev/null 2>&1; then \ + echo "✗ ERROR: kubectl is not installed or not found in PATH"; \ + echo " Please install kubectl: https://kubernetes.io/docs/tasks/tools/#kubectl"; \ + exit 1; \ + fi + @echo "✓ kubectl found: $$($(KUBECTL) version --client 2>/dev/null || echo 'version check failed')" + +.PHONY: istioctl +istioctl: $(ISTIOCTL) ## Download istioctl locally if necessary. +$(ISTIOCTL): $(LOCALBIN) + $(call go-install-tool,$(ISTIOCTL),istio.io/istio/istioctl/cmd/istioctl,$(ISTIOCTL_VERSION)) + +.PHONY: dependency-hash +dependency-hash: ## Calculate hash of dependency versions for caching. + @echo -e "KIND_VERSION=$(KIND_VERSION)\nISTIOCTL_VERSION=$(ISTIOCTL_VERSION)" | sha256sum | cut -d' ' -f1 | head -c 16 + +.PHONY: setup-cluster +setup-cluster: check-kubectl kind istioctl ## Set up a complete kind cluster with cert-manager and Istio. + @export PATH="$(LOCALBIN):$$PATH" && \ + bash scripts/setup-kind.sh && \ + bash scripts/setup-cert-manager.sh && \ + bash scripts/setup-istio.sh + @echo "✓ Cluster setup complete" + +.PHONY: check-kind-context +check-kind-context: check-kubectl ## Verify that the current kubectl context is a kind cluster. + @current_context=$$($(KUBECTL) config current-context 2>/dev/null) || { \ + echo "Error: Unable to get current kubectl context. Is kubectl configured?"; \ + exit 1; \ + }; \ + server_url=$$($(KUBECTL) config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null) || { \ + echo "Error: Unable to get cluster server URL for context '$$current_context'"; \ + exit 1; \ + }; \ + context_check=0; \ + server_check=0; \ + if echo "$$current_context" | grep -qE '^kind-'; then \ + context_check=1; \ + fi; \ + if echo "$$server_url" | grep -qE '(127\.0\.0\.1|localhost)'; then \ + server_check=1; \ + fi; \ + if [ $$context_check -ne 1 ] || [ $$server_check -ne 1 ]; then \ + echo "✗ ERROR: Current context '$$current_context' does not appear to be a kind cluster!"; \ + if [ $$context_check -ne 1 ]; then \ + echo " ✗ Context name does not match kind-* pattern (got: $$current_context)"; \ + fi; \ + if [ $$server_check -ne 1 ]; then \ + echo " ✗ Server URL does not use localhost/127.0.0.1 (got: $$server_url)"; \ + fi; \ + exit 1; \ + fi + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) +endef diff --git a/testing/OWNERS b/testing/OWNERS new file mode 100644 index 00000000..d6cf9cc9 --- /dev/null +++ b/testing/OWNERS @@ -0,0 +1,5 @@ +labels: + - area/ci + - area/v2 +approvers: + - andyatmiami \ No newline at end of file diff --git a/testing/scripts/kind.yml b/testing/scripts/kind.yml new file mode 100644 index 00000000..da9d0c57 --- /dev/null +++ b/testing/scripts/kind.yml @@ -0,0 +1,18 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +# This is needed in order to support projected volumes with service account tokens. +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "service-account-issuer": "kubernetes.default.svc" + "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" +nodes: +- role: control-plane + image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f +- role: worker + image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f \ No newline at end of file diff --git a/testing/scripts/setup-cert-manager.sh b/testing/scripts/setup-cert-manager.sh new file mode 100755 index 00000000..80b657cf --- /dev/null +++ b/testing/scripts/setup-cert-manager.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Setup script for cert-manager +# This script checks if cert-manager is installed and installs it if needed +# Uses the same version as the e2e tests: v1.12.13 (LTS version) + +set -euo pipefail + +# Use LTS version of cert-manager (matches e2e tests) +CERT_MANAGER_VERSION="v1.12.13" +CERT_MANAGER_URL="https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml" + +# Check if cert-manager is already installed +if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then + echo "Cert-manager is already installed" + exit 0 +fi + +echo "Installing cert-manager ${CERT_MANAGER_VERSION}..." +kubectl apply -f "${CERT_MANAGER_URL}" + +echo "Waiting for cert-manager to be ready..." +# Wait for cert-manager webhook to be ready (this is the critical component) +kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/instance=cert-manager \ + -n cert-manager \ + --timeout=120s || { + echo "Warning: cert-manager pods may not be fully ready, but continuing..." +} + +# Also wait for the CRDs to be established +kubectl wait --for=condition=established crd/certificates.cert-manager.io --timeout=60s || true +kubectl wait --for=condition=established crd/issuers.cert-manager.io --timeout=60s || true +kubectl wait --for=condition=established crd/clusterissuers.cert-manager.io --timeout=60s || true + +echo "Cert-manager installation complete" \ No newline at end of file diff --git a/testing/scripts/setup-istio.sh b/testing/scripts/setup-istio.sh new file mode 100755 index 00000000..747b7a1a --- /dev/null +++ b/testing/scripts/setup-istio.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Setup script for Istio +# This script checks if Istio is installed and installs it if needed +# Uses istioctl to install the default profile + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TESTING_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +LOCALBIN="${TESTING_DIR}/bin" + +# Determine istioctl path - prefer LOCALBIN, fallback to PATH +if [ -f "${LOCALBIN}/istioctl" ]; then + ISTIOCTL="${LOCALBIN}/istioctl" +elif command -v istioctl >/dev/null 2>&1; then + ISTIOCTL="istioctl" +else + echo "ERROR: istioctl is not installed. Please install istioctl first:" + echo " cd testing && make istioctl" + echo " or visit: https://istio.io/latest/docs/setup/getting-started/#download" + exit 1 +fi + +# Check if Istio is already installed +# Check for istio-system namespace or istio CRDs +if kubectl get namespace istio-system >/dev/null 2>&1 && \ + kubectl get crd virtualservices.networking.istio.io >/dev/null 2>&1; then + echo "Istio is already installed" + exit 0 +fi + +echo "Installing Istio with default profile..." +"${ISTIOCTL}" install --set profile=default -y + +echo "Waiting for Istio to be ready..." +# Wait for istiod to be ready +kubectl wait --for=condition=ready pod \ + -l app=istiod \ + -n istio-system \ + --timeout=120s || { + echo "Warning: Istio pods may not be fully ready, but continuing..." +} + +# Wait for istio ingress gateway to be ready (if present) +kubectl wait --for=condition=ready pod \ + -l app=istio-ingressgateway \ + -n istio-system \ + --timeout=120s || { + echo "Warning: Istio ingress gateway may not be ready, but continuing..." +} + +echo "Istio installation complete" + diff --git a/testing/scripts/setup-kind.sh b/testing/scripts/setup-kind.sh new file mode 100755 index 00000000..c72ce907 --- /dev/null +++ b/testing/scripts/setup-kind.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Setup script for Kind cluster +# This script checks if a Kind cluster exists and creates it if needed + +set -euo pipefail + +CLUSTER_NAME="local-e2e" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KIND_CONFIG="${SCRIPT_DIR}/kind.yml" + +# Check if kind command exists +if ! command -v kind >/dev/null 2>&1; then + echo "ERROR: kind is not installed. Please install kind first:" + echo " brew install kind # macOS" + echo " or visit: https://kind.sigs.k8s.io/docs/user/quick-start/#installation" + exit 1 +fi + +# Check if cluster exists +if ! kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "Creating Kind cluster '${CLUSTER_NAME}' with config from ${KIND_CONFIG}..." + kind create cluster --name "${CLUSTER_NAME}" --config "${KIND_CONFIG}" --wait 60s + echo "Kind cluster created successfully" +else + echo "Kind cluster '${CLUSTER_NAME}' already exists" +fi + +# Ensure kubectl context is set to the Kind cluster +kubectl config use-context "kind-${CLUSTER_NAME}" || true + +echo "Kind cluster setup complete" \ No newline at end of file diff --git a/workspaces/backend/.gitignore b/workspaces/backend/.gitignore index ce6feea7..eeccbfaa 100644 --- a/workspaces/backend/.gitignore +++ b/workspaces/backend/.gitignore @@ -13,5 +13,8 @@ Dockerfile.cross # Output of the go coverage tool, specifically when used with LiteIDE *.out +# Intermediate output from kustomize +**/kustomize/.output/ + # Go workspace file go.work \ No newline at end of file diff --git a/workspaces/backend/Makefile b/workspaces/backend/Makefile index b3ff69a1..a88c73a8 100644 --- a/workspaces/backend/Makefile +++ b/workspaces/backend/Makefile @@ -145,8 +145,12 @@ $(GOLANGCI_LINT): $(LOCALBIN) .PHONY: deploy deploy: kustomize ## Deploy backend to the K8s cluster specified in ~/.kube/config. - cd manifests/kustomize/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG} - $(KUBECTL) apply -k manifests/kustomize/overlays/istio + @echo "Copying kustomize directory structure to .output..." + @rm -rf manifests/kustomize/.output + @mkdir -p manifests/kustomize/.output + @cp -r manifests/kustomize/* manifests/kustomize/.output/ + @cd manifests/kustomize/.output/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG} + @$(KUBECTL) apply -k manifests/kustomize/.output/overlays/istio .PHONY: undeploy undeploy: kustomize ## Undeploy backend from the K8s cluster specified in ~/.kube/config. diff --git a/workspaces/controller/.gitignore b/workspaces/controller/.gitignore index 89c1ecee..4aee0ca4 100644 --- a/workspaces/controller/.gitignore +++ b/workspaces/controller/.gitignore @@ -13,6 +13,9 @@ Dockerfile.cross # Output of the go coverage tool, specifically when used with LiteIDE *.out +# Intermediate output from kustomize +**/config/.output/ + # Go workspace file go.work diff --git a/workspaces/controller/Makefile b/workspaces/controller/Makefile index 42bb95f6..d36b2689 100644 --- a/workspaces/controller/Makefile +++ b/workspaces/controller/Makefile @@ -140,8 +140,12 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image workspaces-controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - + @echo "Copying config directory structure to .output..." + @rm -rf config/.output + @mkdir -p config/.output + @cp -r config/* config/.output/ + @cd config/.output/manager && $(KUSTOMIZE) edit set image workspaces-controller=${IMG} + @$(KUBECTL) apply -k config/.output/default .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. diff --git a/workspaces/frontend/.gitignore b/workspaces/frontend/.gitignore index 410b6114..d322068e 100644 --- a/workspaces/frontend/.gitignore +++ b/workspaces/frontend/.gitignore @@ -11,3 +11,6 @@ yarn.lock # Test files generated by Jest or Cypress stats.json coverage + +# Intermediate output from kustomize +**/kustomize/.output/ diff --git a/workspaces/frontend/Makefile b/workspaces/frontend/Makefile index ceab7d75..ccf478e7 100755 --- a/workspaces/frontend/Makefile +++ b/workspaces/frontend/Makefile @@ -55,8 +55,12 @@ docker-build-push-multi-arch: .PHONY: deploy deploy: kustomize ## Deploy frontend to the K8s cluster specified in ~/.kube/config. - cd manifests/kustomize/overlays/istio && $(KUSTOMIZE) edit set image workspaces-frontend=${IMG} - $(KUBECTL) apply -k manifests/kustomize/overlays/istio + @echo "Copying kustomize directory structure to .output..." + @rm -rf manifests/kustomize/.output + @mkdir -p manifests/kustomize/.output + @cp -r manifests/kustomize/* manifests/kustomize/.output/ + @cd manifests/kustomize/.output/overlays/istio && $(KUSTOMIZE) edit set image workspaces-frontend=${IMG} + @$(KUBECTL) apply -k manifests/kustomize/.output/overlays/istio .PHONY: undeploy undeploy: kustomize ## Undeploy frontend from the K8s cluster specified in ~/.kube/config.