From aff7e237b7941af917447f1dc662e65d9f568130 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Tue, 7 Nov 2023 15:52:39 +0000 Subject: [PATCH] Adding arm e2e testing for kops cluster --- .github/workflows/e2e-tests.yaml | 17 ++++----- Makefile | 62 +++++++++++++++++++++++++++++++- hack/provenance.sh | 37 +++++++++++++++++++ tests/e2e-kubernetes/kops.sh | 2 +- tests/e2e-kubernetes/run.sh | 25 ++++++++++++- 5 files changed, 132 insertions(+), 11 deletions(-) create mode 100755 hack/provenance.sh diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index f8dd1dd4..a1f3e089 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -27,7 +27,7 @@ jobs: if: github.repository == 'awslabs/mountpoint-s3-csi-driver' strategy: matrix: - cluster-type: ["kops", "eksctl"] + cluster-type: ["kops", "kops-arm"] runs-on: ubuntu-latest permissions: id-token: write @@ -40,10 +40,12 @@ jobs: with: go-version-file: 'go.mod' - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 + with: + version: latest + driver-opts: 'image=moby/buildkit:v0.10.5' - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@master with: @@ -58,16 +60,15 @@ jobs: IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }} run: | BRANCH_OR_TAG=$(echo $GITHUB_REF | cut -d'/' -f3) - export PLATFORM=linux/amd64 + export PLATFORM=linux/amd64,linux/arm64 export TAG=${{ env.COMMIT_ID }} - make build_image - make push_image + make -j `nproc` all-push - name: Install tools run: | export ACTION=install_tools tests/e2e-kubernetes/run.sh - name: Create cluster - if: matrix.cluster-type == 'kops' + if: matrix.cluster-type == 'kops' || matrix.cluster-type == 'kops-arm' run: | export ACTION=create_cluster export AWS_REGION=${{ env.AWS_REGION }} diff --git a/Makefile b/Makefile index a7a3f167..b7371a6c 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,16 @@ IMAGE_NAME?="" IMAGE?=$(REGISTRY)/${IMAGE_NAME} TAG?=$(GIT_COMMIT) +OS?=linux +ARCH?=amd64 +OSVERSION?=amazon + +ALL_OS?=linux +ALL_ARCH_linux?=amd64 arm64 +ALL_OSVERSION_linux?=amazon +ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversion, ${ALL_OSVERSION_linux}, linux-$(arch)-${osversion})) +ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}}) + PLATFORM?=linux/amd64,linux/arm64 # region is expected to be the same where cluster is created @@ -39,11 +49,61 @@ E2E_REGION?=us-east-1 E2E_COMMIT_ID?=local E2E_KUBECONFIG?="" +# split words on hyphen, access by 1-index +word-hyphen = $(word $2,$(subst -, ,$1)) + .EXPORT_ALL_VARIABLES: +# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker) +.PHONY: all +all: all-image-docker + +# Builds all images and pushes them +.PHONY: all-push +all-push: create-manifest-and-images + docker manifest push --purge $(IMAGE):$(TAG) + .PHONY: build_image build_image: - DOCKER_BUILDKIT=1 docker build -t=${IMAGE}:${TAG} --platform=${PLATFORM} . + DOCKER_BUILDKIT=1 docker buildx build -t=${IMAGE}:${TAG} --platform=${PLATFORM} . + +.PHONY: push-manifest +push-manifest: create-manifest + docker manifest push --purge $(IMAGE):$(TAG) + +.PHONY: create-manifest-and-images +create-manifest-and-images: all-image-registry +# sed expression: +# LHS: match 0 or more not space characters +# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS + docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g") + +# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported +# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)." +# https://docs.docker.com/engine/reference/commandline/buildx_build/#output + +.PHONY: all-image-docker +all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux)) +.PHONY: all-image-registry +all-image-registry: $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION)) + +sub-image-%: + $(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image + +.PHONY: image +image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) +.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION): + DOCKER_BUILDKIT=1 docker buildx build \ + --platform=$(OS)/$(ARCH) \ + --progress=plain \ + --target=$(OS)-$(OSVERSION) \ + --output=type=$(OUTPUT_TYPE) \ + -t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \ + --build-arg=GOPROXY=$(GOPROXY) \ + --build-arg=VERSION=$(VERSION) \ + `./hack/provenance.sh` \ + . + touch $@ .PHONY: push_image push_image: diff --git a/hack/provenance.sh b/hack/provenance.sh new file mode 100755 index 00000000..a8b9d344 --- /dev/null +++ b/hack/provenance.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# There is no reliable way to check if a buildx installation supports +# --provenance other than trying to execute it. You cannot even rely +# on the version, because buildx's own installation docs will result +# in installations of buildx that do not correctly report their version +# via `docker buildx version`. +# +# Additionally, if the local buildkit worker is the Docker daemon, +# attestation should not be supported and must be disabled. +# +# Thus, this script echos back the flag `--provenance=false` if and only +# if the local buildx installation supports it. If not, it exits silently. + +BUILDX_TEST=`docker buildx build --provenance=false 2>&1` +if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then + if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then + echo "--provenance=false" + fi +else + echo "Local buildx installation broken?" >&2 + exit 1 +fi \ No newline at end of file diff --git a/tests/e2e-kubernetes/kops.sh b/tests/e2e-kubernetes/kops.sh index 9b2ea826..691f6289 100755 --- a/tests/e2e-kubernetes/kops.sh +++ b/tests/e2e-kubernetes/kops.sh @@ -54,7 +54,7 @@ function kops_create_cluster() { ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" - ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" + ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 20m --kubeconfig "${KUBECONFIG}" } function kops_cluster_exists() { diff --git a/tests/e2e-kubernetes/run.sh b/tests/e2e-kubernetes/run.sh index 0f5a7baa..0d4a5c33 100755 --- a/tests/e2e-kubernetes/run.sh +++ b/tests/e2e-kubernetes/run.sh @@ -35,12 +35,14 @@ KOPS_VERSION=1.28.0 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817 NODE_COUNT=${NODE_COUNT:-3} INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} +INSTANCE_TYPE_ARM=${INSTANCE_TYPE_ARM:-m7g.medium} AMI_ID=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text) +AMI_ID_ARM=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text) CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml} KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml} KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://mountpoint-s3-csi-driver-kops-state-store} - +KOPS_STATE_FILE_ARM=${KOPS_STATE_FILE_ARM:-s3://mountpoint-s3-csi-driver-kops-arm-state-store} HELM_RELEASE_NAME=mountpoint-s3-csi-driver EKSCTL_VERSION=${EKSCTL_VERSION:-0.161.0} @@ -91,6 +93,20 @@ function create_cluster() { "$KOPS_PATCH_FILE" \ "$KOPS_PATCH_NODE_FILE" \ "$KOPS_STATE_FILE" + elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then + kops_create_cluster \ + "$CLUSTER_NAME" \ + "$KOPS_BIN" \ + "$ZONES" \ + "$NODE_COUNT" \ + "$INSTANCE_TYPE_ARM" \ + "$AMI_ID_ARM" \ + "$K8S_VERSION_KOPS" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "$KOPS_PATCH_FILE" \ + "$KOPS_PATCH_NODE_FILE" \ + "$KOPS_STATE_FILE_ARM" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then eksctl_create_cluster \ "$CLUSTER_NAME" \ @@ -111,6 +127,11 @@ function delete_cluster() { "${KOPS_BIN}" \ "${CLUSTER_NAME}" \ "${KOPS_STATE_FILE}" + elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then + kops_delete_cluster \ + "${KOPS_BIN}" \ + "${CLUSTER_NAME}" \ + "${KOPS_STATE_FILE_ARM}" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then eksctl_delete_cluster \ "$EKSCTL_BIN" \ @@ -122,6 +143,8 @@ function delete_cluster() { function update_kubeconfig() { if [[ "${CLUSTER_TYPE}" == "kops" ]]; then ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" + elif [[ "${CLUSTER_TYPE}" == "kops-arm" ]]; then + ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE_ARM}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then aws eks update-kubeconfig --name ${CLUSTER_NAME} --region ${REGION} --kubeconfig=${KUBECONFIG} fi