Adding arm e2e testing for kops cluster

.github/workflows/e2e-tests.yaml

@@ -27,7 +27,7 @@ jobs:
     if: github.repository == 'awslabs/mountpoint-s3-csi-driver'
     strategy:
       matrix:
-        cluster-type: ["kops", "eksctl"]
+        cluster-type: ["kops", "kops-arm"]
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -40,10 +40,12 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+          driver-opts: 'image=moby/buildkit:v0.10.5'
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -58,16 +60,15 @@ jobs:
           IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }}
         run: |
           BRANCH_OR_TAG=$(echo $GITHUB_REF | cut -d'/' -f3)
-          export PLATFORM=linux/amd64
+          export PLATFORM=linux/amd64,linux/arm64
           export TAG=${{ env.COMMIT_ID }}
-          make build_image
-          make push_image
+          make -j `nproc` all-push
       - name: Install tools
         run: |
           export ACTION=install_tools
           tests/e2e-kubernetes/run.sh
       - name: Create cluster
-        if: matrix.cluster-type == 'kops'
+        if: matrix.cluster-type == 'kops' || matrix.cluster-type == 'kops-arm'
         run: |
           export ACTION=create_cluster
           export AWS_REGION=${{ env.AWS_REGION }}

Makefile

@@ -32,18 +32,78 @@ IMAGE_NAME?=""
 IMAGE?=$(REGISTRY)/${IMAGE_NAME}
 TAG?=$(GIT_COMMIT)
 
+OS?=linux
+ARCH?=amd64
+OSVERSION?=amazon
+
+ALL_OS?=linux
+ALL_ARCH_linux?=amd64 arm64
+ALL_OSVERSION_linux?=amazon
+ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversion, ${ALL_OSVERSION_linux}, linux-$(arch)-${osversion}))
+ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}})
+
 PLATFORM?=linux/amd64,linux/arm64
 
 # region is expected to be the same where cluster is created
 E2E_REGION?=us-east-1
 E2E_COMMIT_ID?=local
 E2E_KUBECONFIG?=""
 
+# split words on hyphen, access by 1-index
+word-hyphen = $(word $2,$(subst -, ,$1))
+
 .EXPORT_ALL_VARIABLES:
 
+# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker)
+.PHONY: all
+all: all-image-docker
+
+# Builds all images and pushes them
+.PHONY: all-push
+all-push: create-manifest-and-images
+	docker manifest push --purge $(IMAGE):$(TAG)
+
 .PHONY: build_image
 build_image:
-	DOCKER_BUILDKIT=1 docker build -t=${IMAGE}:${TAG} --platform=${PLATFORM} .
+	DOCKER_BUILDKIT=1 docker buildx build -t=${IMAGE}:${TAG} --platform=${PLATFORM} .
+
+.PHONY: push-manifest
+push-manifest: create-manifest
+	docker manifest push --purge $(IMAGE):$(TAG)
+
+.PHONY: create-manifest-and-images
+create-manifest-and-images: all-image-registry
+# sed expression:
+# LHS: match 0 or more not space characters
+# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
+
+# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported
+# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)."
+# https://docs.docker.com/engine/reference/commandline/buildx_build/#output
+
+.PHONY: all-image-docker
+all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux))
+.PHONY: all-image-registry
+all-image-registry: $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION))
+
+sub-image-%:
+	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image
+
+.PHONY: image
+image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION)
+.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION):
+	DOCKER_BUILDKIT=1 docker buildx build \
+		--platform=$(OS)/$(ARCH) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=$(OUTPUT_TYPE) \
+		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
+		--build-arg=GOPROXY=$(GOPROXY) \
+		--build-arg=VERSION=$(VERSION) \
+		`./hack/provenance.sh` \
+		.
+	touch $@
 
 .PHONY: push_image
 push_image:

hack/provenance.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# There is no reliable way to check if a buildx installation supports
+# --provenance other than trying to execute it. You cannot even rely
+# on the version, because buildx's own installation docs will result
+# in installations of buildx that do not correctly report their version
+# via `docker buildx version`.
+#
+# Additionally, if the local buildkit worker is the Docker daemon,
+# attestation should not be supported and must be disabled.
+#
+# Thus, this script echos back the flag `--provenance=false` if and only
+# if the local buildx installation supports it. If not, it exits silently.
+
+BUILDX_TEST=`docker buildx build --provenance=false 2>&1`
+if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then
+  if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then
+    echo "--provenance=false"
+  fi
+else
+  echo "Local buildx installation broken?" >&2
+  exit 1
+fi

tests/e2e-kubernetes/kops.sh

@@ -54,7 +54,7 @@ function kops_create_cluster() {
   ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
   ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
-  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
+  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 20m --kubeconfig "${KUBECONFIG}"
 }
 
 function kops_cluster_exists() {

tests/e2e-kubernetes/run.sh

@@ -35,12 +35,14 @@ KOPS_VERSION=1.28.0
 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817
 NODE_COUNT=${NODE_COUNT:-3}
 INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
+INSTANCE_TYPE_ARM=${INSTANCE_TYPE_ARM:-m7g.medium}
 AMI_ID=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text)
+AMI_ID_ARM=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text)
 CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml}
 KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://mountpoint-s3-csi-driver-kops-state-store}
-
+KOPS_STATE_FILE_ARM=${KOPS_STATE_FILE_ARM:-s3://mountpoint-s3-csi-driver-kops-arm-state-store}
 HELM_RELEASE_NAME=mountpoint-s3-csi-driver
 
 EKSCTL_VERSION=${EKSCTL_VERSION:-0.161.0}
@@ -91,6 +93,20 @@ function create_cluster() {
       "$KOPS_PATCH_FILE" \
       "$KOPS_PATCH_NODE_FILE" \
       "$KOPS_STATE_FILE"
+  elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then
+    kops_create_cluster \
+      "$CLUSTER_NAME" \
+      "$KOPS_BIN" \
+      "$ZONES" \
+      "$NODE_COUNT" \
+      "$INSTANCE_TYPE_ARM" \
+      "$AMI_ID_ARM" \
+      "$K8S_VERSION_KOPS" \
+      "$CLUSTER_FILE" \
+      "$KUBECONFIG" \
+      "$KOPS_PATCH_FILE" \
+      "$KOPS_PATCH_NODE_FILE" \
+      "$KOPS_STATE_FILE_ARM"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_create_cluster \
       "$CLUSTER_NAME" \
@@ -111,6 +127,11 @@ function delete_cluster() {
       "${KOPS_BIN}" \
       "${CLUSTER_NAME}" \
       "${KOPS_STATE_FILE}"
+  elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then
+    kops_delete_cluster \
+      "${KOPS_BIN}" \
+      "${CLUSTER_NAME}" \
+      "${KOPS_STATE_FILE_ARM}"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_delete_cluster \
       "$EKSCTL_BIN" \
@@ -122,6 +143,8 @@ function delete_cluster() {
 function update_kubeconfig() {
   if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
+  elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then
+    ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE_ARM}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     aws eks update-kubeconfig --name ${CLUSTER_NAME} --region ${REGION} --kubeconfig=${KUBECONFIG}
   fi