From 66448e008788ce1d56f1c01e5f2fff13fc45efba Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Mon, 23 Dec 2024 13:59:40 +0100 Subject: [PATCH 01/10] Add build support for arm64-linux Adds support for arm64-linux. Uses buildx to build multi-platform image. Spawned by: #611 --- .dockerignore | 3 ++- build/Makefile | 22 +++++++++++++++------- image/Dockerfile | 3 ++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index 5a764d40a7..01a7cf4213 100644 --- a/.dockerignore +++ b/.dockerignore @@ -40,6 +40,7 @@ *.exe *.rlib -!target/image/quilkin +!target/image/linux/arm64/quilkin +!target/image/linux/amd64/quilkin !dependencies-src.zip !image/quilkin.yaml \ No newline at end of file diff --git a/build/Makefile b/build/Makefile index aa21533199..e9ec5dfdd7 100644 --- a/build/Makefile +++ b/build/Makefile @@ -57,6 +57,7 @@ gcloud_mount_args := -v $(build_path)/.config/gcloud:/root/.config/gcloud cargo_build_x86_64_linux := build --profile=lto --target x86_64-unknown-linux-gnu cargo_build_x86_64_apple := build --release --target x86_64-apple-darwin cargo_build_aarch64-apple := build --release --target aarch64-apple-darwin +cargo_build_aarch64_linux := build --profile=lto --target aarch64-unknown-linux-gnu cargo_build_x86_64_windows := build --release --target x86_64-pc-windows-gnu # _____ _ @@ -135,14 +136,18 @@ binary-archive: ensure-build-image build-licence-report build-all-binaries docker run --rm $(common_rust_args) -w $(CARGO_TARGET_DIR) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c 'cp ../../license.html . && zip ../../quilkin-$(package_version).zip ./*/lto/quilkin ./*/lto/quilkin.exe ./license.html' -# Build binary for x86_64-unknown-linux-gnu. +# Build binary for x86_64-unknown-linux-gnu and aarch64-unknown-linux-gnu # Use BUILD_LOCAL=1 to build through local cargo rather than through the build container. build-linux-binary: ensure-build-image gen-protobuf ifdef BUILD_LOCAL cargo $(cargo_build_x86_64_linux) + cargo $(cargo_build_aarch64_linux) else - docker run --rm $(common_rust_args) -e "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/x86_64-linux-gnu-gcc" \ - --entrypoint=cargo $(BUILD_IMAGE_TAG) $(cargo_build_x86_64_linux) + docker run --rm $(common_rust_args) \ + -e "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/x86_64-linux-gnu-gcc" \ + -e "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc" \ + $(BUILD_IMAGE_TAG) \ + sh -c "cargo $(cargo_build_x86_64_linux) && cargo $(cargo_build_aarch64_linux) --no-default-features" endif # Build binary for x86_64-pc-windows-gnu @@ -179,15 +184,18 @@ endif # Use BUILD_LOCAL=1 to build the binary through local cargo rather than through the build container. build-image: ensure-build-image build-licence-report build-linux-binary build-image: - -mkdir -p "$(project_path)/target/image/" + -mkdir -p "$(project_path)/target/image/linux/amd64" + -mkdir -p "$(project_path)/target/image/linux/arm64" ifdef BUILD_LOCAL - cp "$(project_path)/target/x86_64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/" + cp "$(project_path)/target/x86_64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/linux/amd64/" + cp "$(project_path)/target/aarch64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/linux/arm64/" else - cp "$(project_path)/target/build-image/x86_64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/" + cp "$(project_path)/target/build-image/x86_64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/linux/amd64/" + cp "$(project_path)/target/build-image/aarch64-unknown-linux-gnu/lto/quilkin" "$(project_path)/target/image/linux/arm64/" endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' - docker build --platform=linux/amd64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) + docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) # Generates the HTML report of all open source licence dependencies build-licence-report: ensure-build-image diff --git a/image/Dockerfile b/image/Dockerfile index 298b7f66cd..33cafb75e8 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -14,9 +14,10 @@ FROM gcr.io/distroless/cc-debian12:nonroot as base WORKDIR / +ARG TARGETPLATFORM COPY ./license.html . COPY ./dependencies-src.zip . -COPY --chown=nonroot:nonroot ./target/image/quilkin . +COPY --chown=nonroot:nonroot ./target/image/${TARGETPLATFORM}/quilkin . USER nonroot:nonroot ENTRYPOINT ["/quilkin"] From 7f0262aea75bae01271b0098416d77f66734be0f Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Tue, 7 Jan 2025 15:46:33 +0100 Subject: [PATCH 02/10] Add explicitly setting CC for aarch64-linux builds to fix cross-compilation issue with Ring --- build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Makefile b/build/Makefile index e9ec5dfdd7..311ba270fe 100644 --- a/build/Makefile +++ b/build/Makefile @@ -147,7 +147,7 @@ else -e "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/x86_64-linux-gnu-gcc" \ -e "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc" \ $(BUILD_IMAGE_TAG) \ - sh -c "cargo $(cargo_build_x86_64_linux) && cargo $(cargo_build_aarch64_linux) --no-default-features" + sh -c "cargo $(cargo_build_x86_64_linux) && CC=aarch64-linux-gnu-gcc cargo $(cargo_build_aarch64_linux) --no-default-features" endif # Build binary for x86_64-pc-windows-gnu From fc8dad51168840b6fc77f27fb1081c1c67a83693 Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Tue, 7 Jan 2025 21:06:12 +0100 Subject: [PATCH 03/10] Add package g++-aarch64-linux-gnu to quilkin-build --- build/build-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index 5f00d6e09e..d76f6f4373 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -28,7 +28,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \ RUN set -eux && \ apt-get update && \ apt-get install -y lsb-release jq curl wget zip git build-essential software-properties-common protobuf-compiler \ - libssl-dev pkg-config nodejs npm bash-completion g++-x86-64-linux-gnu g++-mingw-w64-x86-64 && \ + libssl-dev pkg-config nodejs npm bash-completion g++-x86-64-linux-gnu g++-mingw-w64-x86-64 g++-aarch64-linux-gnu && \ npm install -g browser-sync && \ echo "source /etc/bash_completion" >> /root/.bashrc From cd6b5a1367fc1e6789fea402093c51dfdd4685e2 Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Wed, 8 Jan 2025 17:27:17 +0100 Subject: [PATCH 04/10] Update multiarch build to use docker-container as build driver --- build/Makefile | 3 ++- image/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/Makefile b/build/Makefile index 311ba270fe..738a5e802d 100644 --- a/build/Makefile +++ b/build/Makefile @@ -195,7 +195,8 @@ else endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' - docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) + docker buildx create --use --driver=docker-container --name container --platform=linux/amd64,linux/arm64 + docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container # Generates the HTML report of all open source licence dependencies build-licence-report: ensure-build-image diff --git a/image/Dockerfile b/image/Dockerfile index 33cafb75e8..d65e10c053 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/distroless/cc-debian12:nonroot as base +FROM gcr.io/distroless/cc-debian12:nonroot AS base WORKDIR / ARG TARGETPLATFORM COPY ./license.html . From 78cea0a2e6c34db6db1ae49040801b063ef45318 Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Wed, 8 Jan 2025 19:20:15 +0100 Subject: [PATCH 05/10] Add --load to load image into Docker --- build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Makefile b/build/Makefile index 738a5e802d..b1cc9bcf88 100644 --- a/build/Makefile +++ b/build/Makefile @@ -196,7 +196,7 @@ endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' docker buildx create --use --driver=docker-container --name container --platform=linux/amd64,linux/arm64 - docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container + docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container --load # Generates the HTML report of all open source licence dependencies build-licence-report: ensure-build-image From 0aa6b8353ab5314b6476bb964b898403e71da16e Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Thu, 9 Jan 2025 09:24:56 +0100 Subject: [PATCH 06/10] Added --push to docker build command Restructured test-agones to require build-image directly since it's already doing the pushing. --- build/Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build/Makefile b/build/Makefile index b1cc9bcf88..7b36ec18cb 100644 --- a/build/Makefile +++ b/build/Makefile @@ -196,7 +196,7 @@ endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' docker buildx create --use --driver=docker-container --name container --platform=linux/amd64,linux/arm64 - docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container --load + docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container --push # Generates the HTML report of all open source licence dependencies build-licence-report: ensure-build-image @@ -211,7 +211,9 @@ build-licence-report: # to set those options. # If a `kubectl` authentication failure occurs, run `kubectl get ns` to confirm access and refresh the Kubernetes # authentication token, and try again if successful. -test-agones: push +ifndef SKIP_BUILD_IMAGE +test-agones: build-image +endif test-agones: $(MAKE) run-test-agones @@ -233,9 +235,6 @@ run-test-agones: # USe `SKIP_BUILD_IMAGE` if you want to skip building the image if it has been already built. # See `build-image` for more details. push: -ifndef SKIP_BUILD_IMAGE -push: build-image -endif docker push $(IMAGE_TAG) # Convenience target to build and push quilkin images into a minikube instance From 55deff741ca137e531c16552b36f55437f23d995 Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Fri, 17 Jan 2025 13:04:13 +0100 Subject: [PATCH 07/10] Explicitly creating docker manifest for multi-arch build --- build/Makefile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build/Makefile b/build/Makefile index 7b36ec18cb..5c6755e923 100644 --- a/build/Makefile +++ b/build/Makefile @@ -36,8 +36,10 @@ else endif # Set this value if you want to use an external registry -REPOSITORY ?= "" +REPOSITORY ?= "tocakaboing/" IMAGE_TAG ?= ${REPOSITORY}quilkin:$(package_version) +IMAGE_TAG_ARM64 ?= ${REPOSITORY}quilkin-arm64:$(package_version) +IMAGE_TAG_AMD64 ?= ${REPOSITORY}quilkin-amd64:$(package_version) PREV_IMAGE_TAG ?= us-docker.pkg.dev/quilkin/release/quilkin:0.8.0 MINIKUBE_PROFILE ?= quilkin CARGO_TARGET_DIR ?= /workspace/target/build-image @@ -195,8 +197,12 @@ else endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' - docker buildx create --use --driver=docker-container --name container --platform=linux/amd64,linux/arm64 - docker buildx build --platform=linux/amd64,linux/arm64 -t $(IMAGE_TAG) -f $(project_path)/image/Dockerfile $(project_path) --builder=container --push + docker buildx build --platform=linux/amd64 --build-arg TARGETPLATFORM=linux/amd64 --provenance false -t $(IMAGE_TAG_AMD64) -f $(project_path)/image/Dockerfile $(project_path) + docker buildx build --platform=linux/arm64 --build-arg TARGETPLATFORM=linux/arm64 --provenance false -t $(IMAGE_TAG_ARM64) -f $(project_path)/image/Dockerfile $(project_path) + docker push $(IMAGE_TAG_AMD64) + docker push $(IMAGE_TAG_ARM64) + docker manifest create $(IMAGE_TAG) $(IMAGE_TAG_AMD64) $(IMAGE_TAG_ARM64) + docker manifest push $(IMAGE_TAG) # Generates the HTML report of all open source licence dependencies build-licence-report: ensure-build-image From 09bbe46f216f60598ecab9bf42ce808f030adacb Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Fri, 17 Jan 2025 13:05:24 +0100 Subject: [PATCH 08/10] Explicitly create manifest and push --- build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Makefile b/build/Makefile index 5c6755e923..34853f7aa8 100644 --- a/build/Makefile +++ b/build/Makefile @@ -36,7 +36,7 @@ else endif # Set this value if you want to use an external registry -REPOSITORY ?= "tocakaboing/" +REPOSITORY ?= "" IMAGE_TAG ?= ${REPOSITORY}quilkin:$(package_version) IMAGE_TAG_ARM64 ?= ${REPOSITORY}quilkin-arm64:$(package_version) IMAGE_TAG_AMD64 ?= ${REPOSITORY}quilkin-amd64:$(package_version) From 306c9634a4174f9955e76b6895e815841070245e Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Wed, 22 Jan 2025 13:40:22 +0100 Subject: [PATCH 09/10] Potential fix for docker manifest list tagging issue Minor fix for quoting issue on package_version --- build/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Makefile b/build/Makefile index 34853f7aa8..7d1915ba8f 100644 --- a/build/Makefile +++ b/build/Makefile @@ -29,10 +29,10 @@ rust_toolchain := $(shell grep channel $(project_path)/rust-toolchain.toml | awk # if this is a release, don't put the sha, otherwise, leave it off. ifdef QUILKIN_RELEASE - package_version := $(shell grep -A1 -w "name = \"quilkin\"" $(project_path)/Cargo.toml | grep version -m 1 | awk '{print $$3}') + package_version := $(shell grep -A1 -w "name = \"quilkin\"" $(project_path)/Cargo.toml | grep version -m 1 | awk -F'"' '{print $$2}') else git_sha := $(shell git rev-parse --short=7 HEAD) - package_version := $(shell grep -A1 -w "name = \"quilkin\"" $(project_path)/Cargo.toml | grep version -m 1 | awk '{print $$3}')-${git_sha} + package_version := $(shell grep -A1 -w "name = \"quilkin\"" $(project_path)/Cargo.toml | grep version -m 1 | awk -F'"' '{print $$2}')-${git_sha} endif # Set this value if you want to use an external registry @@ -197,8 +197,8 @@ else endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' - docker buildx build --platform=linux/amd64 --build-arg TARGETPLATFORM=linux/amd64 --provenance false -t $(IMAGE_TAG_AMD64) -f $(project_path)/image/Dockerfile $(project_path) - docker buildx build --platform=linux/arm64 --build-arg TARGETPLATFORM=linux/arm64 --provenance false -t $(IMAGE_TAG_ARM64) -f $(project_path)/image/Dockerfile $(project_path) + docker build --platform=linux/amd64 --build-arg TARGETPLATFORM=linux/amd64 -t $(IMAGE_TAG_AMD64) -f $(project_path)/image/Dockerfile $(project_path) + docker build --platform=linux/arm64 --build-arg TARGETPLATFORM=linux/arm64 -t $(IMAGE_TAG_ARM64) -f $(project_path)/image/Dockerfile $(project_path) docker push $(IMAGE_TAG_AMD64) docker push $(IMAGE_TAG_ARM64) docker manifest create $(IMAGE_TAG) $(IMAGE_TAG_AMD64) $(IMAGE_TAG_ARM64) From 1616d8c5ec6d2e5b634633f7ca7ad3e3bba3b3d9 Mon Sep 17 00:00:00 2001 From: Jonas Lincoln Date: Wed, 22 Jan 2025 14:59:46 +0100 Subject: [PATCH 10/10] Reinstate buildx builds again --- build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Makefile b/build/Makefile index 7d1915ba8f..4db64a6acb 100644 --- a/build/Makefile +++ b/build/Makefile @@ -197,8 +197,8 @@ else endif docker run --rm $(common_rust_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh' - docker build --platform=linux/amd64 --build-arg TARGETPLATFORM=linux/amd64 -t $(IMAGE_TAG_AMD64) -f $(project_path)/image/Dockerfile $(project_path) - docker build --platform=linux/arm64 --build-arg TARGETPLATFORM=linux/arm64 -t $(IMAGE_TAG_ARM64) -f $(project_path)/image/Dockerfile $(project_path) + docker buildx build --platform=linux/amd64 --build-arg TARGETPLATFORM=linux/amd64 -t $(IMAGE_TAG_AMD64) -f $(project_path)/image/Dockerfile $(project_path) + docker buildx build --platform=linux/arm64 --build-arg TARGETPLATFORM=linux/arm64 -t $(IMAGE_TAG_ARM64) -f $(project_path)/image/Dockerfile $(project_path) docker push $(IMAGE_TAG_AMD64) docker push $(IMAGE_TAG_ARM64) docker manifest create $(IMAGE_TAG) $(IMAGE_TAG_AMD64) $(IMAGE_TAG_ARM64)