Adding ARM testing in E2E test suite (#46)

* Adding arm to e2e testing

.github/workflows/e2e-tests.yaml

@@ -28,6 +28,7 @@ jobs:
     strategy:
       matrix:
         cluster-type: ["kops", "eksctl"]
+        arch: ["x86", "arm"]
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -40,10 +41,9 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -58,10 +58,9 @@ jobs:
           IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }}
         run: |
           BRANCH_OR_TAG=$(echo $GITHUB_REF | cut -d'/' -f3)
-          export PLATFORM=linux/amd64
+          export PLATFORM=linux/amd64,linux/arm64
           export TAG=${{ env.COMMIT_ID }}
-          make build_image
-          make push_image
+          make -j `nproc` all-push
       - name: Install tools
         run: |
           export ACTION=install_tools
@@ -72,12 +71,14 @@ jobs:
           export ACTION=create_cluster
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/run.sh
       - name: Update kubeconfig
         run: |
           export ACTION=update_kubeconfig
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/run.sh
       - name: Install the driver
         run: |
@@ -86,6 +87,7 @@ jobs:
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export IMAGE_NAME=${{ env.TMP_IMAGE_NAME }}
           export TAG=${{ env.COMMIT_ID }}
+          export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/run.sh
       - name: Run E2E Tests
         run: |
@@ -94,20 +96,23 @@ jobs:
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export TAG=${{ env.COMMIT_ID }}
+          export ARCH=${{ matrix.arch }}
           ./run.sh
-      - name: Uinstall the driver
+      - name: Uninstall the driver
         if: always()
         run: |
           export ACTION=uninstall_driver
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/run.sh
       - name: Delete cluster
         if: always() && matrix.cluster-type == 'kops'
         run: |
           export ACTION=delete_cluster
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/run.sh
       - name: Promote image for release branch
         if: ${{ startsWith(github.ref_name, 'release') && matrix.cluster-type == 'eksctl' }}

Dockerfile

@@ -17,11 +17,12 @@ ARG MOUNTPOINT_VERSION=1.1.0
 # Download and verify the mountpoint's RPM and DEB in this container
 FROM --platform=$BUILDPLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 as mp_builder
 ARG MOUNTPOINT_VERSION
-
+ARG TARGETARCH
+ARG TARGETPLATFORM
 # We need the full version of GnuPG
 RUN dnf install -y --allowerasing wget gnupg2
 
-RUN MP_ARCH=`uname -p | sed s/aarch64/arm64/` && \
+RUN MP_ARCH=`echo ${TARGETARCH} | sed s/amd64/x86_64/` && \
     wget -q "https://s3.amazonaws.com/mountpoint-s3-release/${MOUNTPOINT_VERSION}/$MP_ARCH/mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.rpm" && \
     wget -q "https://s3.amazonaws.com/mountpoint-s3-release/${MOUNTPOINT_VERSION}/$MP_ARCH/mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.rpm.asc" && \
     wget -q "https://s3.amazonaws.com/mountpoint-s3-release/${MOUNTPOINT_VERSION}/$MP_ARCH/mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.deb" && \
@@ -33,35 +34,29 @@ RUN gpg --import KEYS && \
     (gpg --fingerprint [email protected] | grep "673F E406 1506 BB46 9A0E  F857 BE39 7A52 B086 DA5A")
 
 # Verify the downloaded binary
-RUN MP_ARCH=`uname -p | sed s/aarch64/arm64/` && \
+RUN MP_ARCH=`echo ${TARGETARCH} | sed s/amd64/x86_64/` && \
     gpg --verify mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.rpm.asc && \
     gpg --verify mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.deb.asc && \
     mv mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.rpm /mount-s3.rpm && \
     mv mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.deb /mount-s3.deb
 
 # Build driver
 FROM --platform=$BUILDPLATFORM golang:1.21.1-bullseye as builder
-
+ARG TARGETARCH
 WORKDIR /go/src/github.com/awslabs/mountpoint-s3-csi-driver
 COPY . .
 RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
-make bin
+TARGETARCH=${TARGETARCH} make bin
 
-FROM --platform=$BUILDPLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS linux-amazon
+FROM --platform=$TARGETPLATFORM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi:latest-al2 AS linux-amazon
 ARG MOUNTPOINT_VERSION
 ENV MOUNTPOINT_VERSION=${MOUNTPOINT_VERSION}
 
-RUN yum install util-linux -y
-
 # MP Installer
 COPY --from=mp_builder /mount-s3.rpm /mount-s3.rpm
 COPY --from=mp_builder /mount-s3.deb /mount-s3.deb
 COPY ./cmd/install-mp.sh /install-mp.sh
 
-RUN dnf upgrade -y && \
-    dnf install -y ./mount-s3.rpm && \
-    dnf clean all
-
 # Install driver
 COPY --from=builder /go/src/github.com/awslabs/mountpoint-s3-csi-driver/bin/aws-s3-csi-driver /bin/aws-s3-csi-driver
 

Makefile

@@ -32,18 +32,78 @@ IMAGE_NAME?=""
 IMAGE?=$(REGISTRY)/${IMAGE_NAME}
 TAG?=$(GIT_COMMIT)
 
+OS?=linux
+ARCH?=amd64
+OSVERSION?=amazon
+
+ALL_OS?=linux
+ALL_ARCH_linux?=amd64 arm64
+ALL_OSVERSION_linux?=amazon
+ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversion, ${ALL_OSVERSION_linux}, linux-$(arch)-${osversion}))
+ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}})
+
 PLATFORM?=linux/amd64,linux/arm64
 
 # region is expected to be the same where cluster is created
 E2E_REGION?=us-east-1
 E2E_COMMIT_ID?=local
 E2E_KUBECONFIG?=""
 
+# split words on hyphen, access by 1-index
+word-hyphen = $(word $2,$(subst -, ,$1))
+
 .EXPORT_ALL_VARIABLES:
 
+# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker)
+.PHONY: all
+all: all-image-docker
+
+# Builds all images and pushes them
+.PHONY: all-push
+all-push: create-manifest-and-images
+	docker manifest push --purge $(IMAGE):$(TAG)
+
 .PHONY: build_image
 build_image:
-	DOCKER_BUILDKIT=1 docker build -t=${IMAGE}:${TAG} --platform=${PLATFORM} .
+	DOCKER_BUILDKIT=1 docker buildx build -t=${IMAGE}:${TAG} --platform=${PLATFORM} .
+
+.PHONY: push-manifest
+push-manifest: create-manifest
+	docker manifest push --purge $(IMAGE):$(TAG)
+
+.PHONY: create-manifest-and-images
+create-manifest-and-images: all-image-registry
+# sed expression:
+# LHS: match 0 or more not space characters
+# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
+
+# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported
+# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)."
+# https://docs.docker.com/engine/reference/commandline/buildx_build/#output
+
+.PHONY: all-image-docker
+all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux))
+.PHONY: all-image-registry
+all-image-registry: $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION))
+
+sub-image-%:
+	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image
+
+.PHONY: image
+image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION)
+.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION):
+	DOCKER_BUILDKIT=1 docker buildx build \
+		--platform=$(OS)/$(ARCH) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=$(OUTPUT_TYPE) \
+		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
+		--build-arg=GOPROXY=$(GOPROXY) \
+		--build-arg=VERSION=$(VERSION) \
+		`./scripts/provenance.sh` \
+		.
+	touch $@
 
 .PHONY: push_image
 push_image:
@@ -56,7 +116,7 @@ login_registry:
 .PHONY: bin
 bin:
 	mkdir -p bin
-	CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -o bin/aws-s3-csi-driver ./cmd/
+	CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags ${LDFLAGS} -o bin/aws-s3-csi-driver ./cmd/
 
 .PHONY: test
 test:

deploy/kubernetes/overlays/stable/kustomization.yaml

@@ -6,3 +6,4 @@ images:
 - name: csi-driver
   newName: 151381207180.dkr.ecr.eu-west-1.amazonaws.com/s3-csi-driver
   newTag: v0.1.0
+

scripts/provenance.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# There is no reliable way to check if a buildx installation supports
+# --provenance other than trying to execute it. You cannot even rely
+# on the version, because buildx's own installation docs will result
+# in installations of buildx that do not correctly report their version
+# via `docker buildx version`.
+#
+# Additionally, if the local buildkit worker is the Docker daemon,
+# attestation should not be supported and must be disabled.
+#
+# Thus, this script echos back the flag `--provenance=false` if and only
+# if the local buildx installation supports it. If not, it exits silently.
+
+BUILDX_TEST=`docker buildx build --provenance=false 2>&1`
+if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then
+  if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then
+    echo "--provenance=false"
+  fi
+else
+  echo "Local buildx installation broken?" >&2
+  exit 1
+fi

tests/e2e-kubernetes/eksctl.sh

@@ -22,13 +22,15 @@ function eksctl_create_cluster() {
   EKSCTL_PATCH_FILE=${7}
   ZONES=${8}
   CI_ROLE_ARN=${9}
+  NODE_TYPE=${10}
 
   eksctl_delete_cluster "$BIN" "$CLUSTER_NAME" "$REGION"
 
   # CAUTION: this may fail with "the targeted availability zone, does not currently have sufficient capacity to support the cluster" error, we may require a fix for that
   ${BIN} create cluster \
     --name $CLUSTER_NAME \
     --region $REGION \
+    --node-type $NODE_TYPE \
     --with-oidc \
     --zones $ZONES \
     --dry-run > $CLUSTER_FILE

tests/e2e-kubernetes/kops.sh

@@ -54,7 +54,7 @@ function kops_create_cluster() {
   ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
   ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
-  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
+  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 15m --kubeconfig "${KUBECONFIG}"
 }
 
 function kops_cluster_exists() {