From de4a046372bf546e341ee225ff355a287be7c352 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Sat, 8 Feb 2025 06:31:55 +0000 Subject: [PATCH 1/2] Assign alias tag even if PUBLISH_CONTAINER=0 --- containers/docker_build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/containers/docker_build.sh b/containers/docker_build.sh index 07f7376..09fa7c6 100755 --- a/containers/docker_build.sh +++ b/containers/docker_build.sh @@ -93,6 +93,12 @@ python3 containers/docker_build.py \ --container-def ${CONTAINER_DEF} \ --container-tag ${CONTAINER_TAG} \ ${BUILD_ARGS} + +# Create another alias for the container using the branch name +CONTAINER_ALIAS="${ECR_URL}/${CONTAINER_ID}:${BRANCH_NAME}" +echo "docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS}" +docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS} + set +x # Now push the new container to ECR @@ -116,10 +122,6 @@ then exit 1 fi - # Create another alias for the container using the branch name - CONTAINER_ALIAS="${ECR_URL}/${CONTAINER_ID}:${BRANCH_NAME}" - echo "docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS}" - docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS} echo "docker push --quiet ${CONTAINER_ALIAS}" docker push --quiet "${CONTAINER_ALIAS}" fi From 23d6671358506fe30c10948372d1827b48751c44 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Sat, 8 Feb 2025 06:52:55 +0000 Subject: [PATCH 2/2] Use precise terminology for image components --- .github/workflows/containers.yml | 14 +++++----- containers/docker_build.py | 10 +++---- containers/docker_build.sh | 46 ++++++++++++++++---------------- containers/extract_build_args.jq | 4 +-- containers/extract_build_args.sh | 12 ++++----- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 2fc25b1..e95556d 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -33,16 +33,16 @@ env: jobs: build-containers: - name: Build CI containers (${{ matrix.container_id }}) + name: Build CI containers (${{ matrix.image_repo }}) runs-on: - runs-on - runner=${{ matrix.runner }} - run-id=${{ github.run_id }} - - tag=build-containers-${{ matrix.container_id }} + - tag=build-containers-${{ matrix.image_repo }} strategy: fail-fast: false matrix: - container_id: + image_repo: - xgb-ci.clang_tidy - xgb-ci.cpu - xgb-ci.cpu_build_r_doc @@ -57,9 +57,9 @@ jobs: - xgb-ci.i386 runner: [linux-amd64-cpu] include: - - container_id: xgb-ci.aarch64 + - image_repo: xgb-ci.aarch64 runner: linux-arm64-cpu - - container_id: xgb-ci.manylinux2014_aarch64 + - image_repo: xgb-ci.manylinux2014_aarch64 runner: linux-arm64-cpu steps: - name: Workflow trigger information @@ -71,5 +71,5 @@ jobs: - uses: actions/checkout@v4 with: submodules: "true" - - name: Build ${{ matrix.container_id }} - run: bash containers/docker_build.sh ${{ matrix.container_id }} + - name: Build ${{ matrix.image_repo }} + run: bash containers/docker_build.sh ${{ matrix.image_repo }} diff --git a/containers/docker_build.py b/containers/docker_build.py index 91a5657..00cba40 100644 --- a/containers/docker_build.py +++ b/containers/docker_build.py @@ -47,7 +47,7 @@ def parse_build_args(*, raw_build_args: list[str]) -> dict[str, str]: def docker_build( *, - container_tag: str, + image_uri: str, build_args: dict[str, str], dockerfile_path: pathlib.Path, docker_context_path: pathlib.Path, @@ -70,7 +70,7 @@ def docker_build( "--ulimit", "nofile=1024000:1024000", "-t", - container_tag, + image_uri, "-f", str(dockerfile_path), str(docker_context_path), @@ -90,7 +90,7 @@ def main(*, args: argparse.Namespace) -> None: build_args = parse_build_args(raw_build_args=args.build_arg) docker_build( - container_tag=args.container_tag, + image_uri=args.image_uri, build_args=build_args, dockerfile_path=dockerfile_path, docker_context_path=docker_context_path, @@ -110,11 +110,11 @@ def main(*, args: argparse.Namespace) -> None: ), ) parser.add_argument( - "--container-tag", + "--image-uri", type=str, required=True, help=( - "Tag to assign to the newly built container, e.g. " + "Fully qualified image URI to identify the container, e.g. " "492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main" ), ) diff --git a/containers/docker_build.sh b/containers/docker_build.sh index 09fa7c6..e2c802d 100755 --- a/containers/docker_build.sh +++ b/containers/docker_build.sh @@ -8,9 +8,9 @@ USAGE_DOC=$( cat <<-EOF -Usage: containers/docker_build.sh [container_id] +Usage: containers/docker_build.sh IMAGE_REPO -where [container_id] is used to fetch the container definition and build-time variables +where IMAGE_REPO is used to fetch the container definition and build-time variables from containers/ci_container.yml. In addition, the following environment variables should be set. @@ -26,7 +26,7 @@ EOF # Configure ECR to delete containers older than 30 days. ECR_AWS_ACCOUNT_ID="492475357299" ECR_AWS_REGION="us-west-2" -ECR_URL="${ECR_AWS_ACCOUNT_ID}.dkr.ecr.${ECR_AWS_REGION}.amazonaws.com" +IMAGE_REGISTRY="${ECR_AWS_ACCOUNT_ID}.dkr.ecr.${ECR_AWS_REGION}.amazonaws.com" ECR_LIFECYCLE_RULE=$( cat <<-EOF { @@ -64,10 +64,10 @@ then echo "${USAGE_DOC}" exit 2 fi -CONTAINER_ID="$1" +IMAGE_REPO="$1" # Fetch CONTAINER_DEF and BUILD_ARGS -source <(containers/extract_build_args.sh ${CONTAINER_ID} | tee /dev/stderr) 2>&1 +source <(containers/extract_build_args.sh ${IMAGE_REPO} | tee /dev/stderr) 2>&1 if [[ "${PUBLISH_CONTAINER:-}" != "1" ]] # Any value other than 1 is considered false then @@ -78,26 +78,26 @@ if [[ ${PUBLISH_CONTAINER} -eq 0 ]] then echo "PUBLISH_CONTAINER not set; the container will not be published" else - echo "The container will be published at ${ECR_URL}" + echo "The container will be published at ${IMAGE_REGISTRY}" # Login for Docker registry echo "aws ecr get-login-password --region ${ECR_AWS_REGION} |" \ - "docker login --username AWS --password-stdin ${ECR_URL}" + "docker login --username AWS --password-stdin ${IMAGE_REGISTRY}" aws ecr get-login-password --region ${ECR_AWS_REGION} \ - | docker login --username AWS --password-stdin ${ECR_URL} + | docker login --username AWS --password-stdin ${IMAGE_REGISTRY} fi # Run Docker build set -x -CONTAINER_TAG="${ECR_URL}/${CONTAINER_ID}:${GITHUB_SHA}" +IMAGE_URI="${IMAGE_REGISTRY}/${IMAGE_REPO}:${GITHUB_SHA}" python3 containers/docker_build.py \ --container-def ${CONTAINER_DEF} \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ ${BUILD_ARGS} -# Create another alias for the container using the branch name -CONTAINER_ALIAS="${ECR_URL}/${CONTAINER_ID}:${BRANCH_NAME}" -echo "docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS}" -docker tag ${CONTAINER_TAG} ${CONTAINER_ALIAS} +# Create an alias for the container using the branch name as tag +IMAGE_URI_ALIAS="${IMAGE_REGISTRY}/${IMAGE_REPO}:${BRANCH_NAME}" +echo "docker tag ${IMAGE_URI} ${IMAGE_URI_ALIAS}" +docker tag ${IMAGE_URI} ${IMAGE_URI_ALIAS} set +x @@ -105,23 +105,23 @@ set +x if [[ ${PUBLISH_CONTAINER} -eq 1 ]] then # Attempt to create Docker repository; it will fail if the repository already exists - echo "aws ecr create-repository --repository-name ${CONTAINER_ID} --region ${ECR_AWS_REGION}" - if aws ecr create-repository --repository-name ${CONTAINER_ID} --region ${ECR_AWS_REGION} + echo "aws ecr create-repository --repository-name ${IMAGE_REPO} --region ${ECR_AWS_REGION}" + if aws ecr create-repository --repository-name ${IMAGE_REPO} --region ${ECR_AWS_REGION} then # Repository was created. Now set expiration policy - echo "aws ecr put-lifecycle-policy --repository-name ${CONTAINER_ID}" \ + echo "aws ecr put-lifecycle-policy --repository-name ${IMAGE_REPO}" \ "--region ${ECR_AWS_REGION} --lifecycle-policy-text file:///dev/stdin" - echo "${ECR_LIFECYCLE_RULE}" | aws ecr put-lifecycle-policy --repository-name ${CONTAINER_ID} \ + echo "${ECR_LIFECYCLE_RULE}" | aws ecr put-lifecycle-policy --repository-name ${IMAGE_REPO} \ --region ${ECR_AWS_REGION} --lifecycle-policy-text file:///dev/stdin fi - echo "docker push --quiet ${CONTAINER_TAG}" - if ! time docker push --quiet "${CONTAINER_TAG}" + echo "docker push --quiet ${IMAGE_URI}" + if ! time docker push --quiet "${IMAGE_URI}" then - echo "ERROR: could not update Docker cache ${CONTAINER_TAG}" + echo "ERROR: could not update Docker cache ${IMAGE_URI}" exit 1 fi - echo "docker push --quiet ${CONTAINER_ALIAS}" - docker push --quiet "${CONTAINER_ALIAS}" + echo "docker push --quiet ${IMAGE_URI_ALIAS}" + docker push --quiet "${IMAGE_URI_ALIAS}" fi diff --git a/containers/extract_build_args.jq b/containers/extract_build_args.jq index b35240e..d6e405b 100644 --- a/containers/extract_build_args.jq +++ b/containers/extract_build_args.jq @@ -2,9 +2,9 @@ ## xgb-ci.gpu_build_r_rockylinux8 ## Example output: ## --build-arg CUDA_VERSION_ARG=12.4.1 --build-arg R_VERSION_ARG=4.3.2 -def compute_build_args($input; $container_id): +def compute_build_args($input; $image_repo): $input | - .[$container_id] | + .[$image_repo] | select(.build_args != null) | .build_args | to_entries | diff --git a/containers/extract_build_args.sh b/containers/extract_build_args.sh index 4254915..2d4985a 100755 --- a/containers/extract_build_args.sh +++ b/containers/extract_build_args.sh @@ -1,6 +1,6 @@ #!/bin/bash ## Extract container definition and build args from containers/ci_container.yml, -## given the container ID. +## given the image repo. ## ## Example input: ## xgb-ci.clang_tidy @@ -8,19 +8,19 @@ ## CONTAINER_DEF='clang_tidy' BUILD_ARGS='--build-arg CUDA_VERSION_ARG=12.4.1' if [ "$#" -ne 1 ]; then - echo "Usage: $0 [container_id]" + echo "Usage: $0 [image_repo]" exit 1 fi -CONTAINER_ID="$1" +IMAGE_REPO="$1" CONTAINER_DEF=$( yq -o json containers/ci_container.yml | - jq -r --arg container_id "${CONTAINER_ID}" '.[$container_id].container_def' + jq -r --arg image_repo "${IMAGE_REPO}" '.[$image_repo].container_def' ) BUILD_ARGS=$( yq -o json containers/ci_container.yml | - jq -r --arg container_id "${CONTAINER_ID}" \ + jq -r --arg image_repo "${IMAGE_REPO}" \ 'include "containers/extract_build_args"; - compute_build_args(.; $container_id)' + compute_build_args(.; $image_repo)' ) echo "CONTAINER_DEF='${CONTAINER_DEF}' BUILD_ARGS='${BUILD_ARGS}'"