From 1573d5385425e438ae935ffb6e213277f0bc8f03 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:17:51 -0700 Subject: [PATCH 1/9] use pinned manylinux image version in cibuildwheel --- .github/workflows/build-wheels.yml | 1 + .../manylinux_2_28-openssl.Dockerfile | 2 +- .../update-manylinux-openssl-image.yml | 43 ++++++++++++++++--- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 2568581682..da6720cfb3 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -294,6 +294,7 @@ jobs: - name: Build wheel uses: pypa/cibuildwheel@v2.21.3 + id: cibuildwheel env: # manylinux_2_28 x64 image doesn't search in this directory for shared libraries CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH=/usr/local/lib64 diff --git a/.github/workflows/manylinux_2_28-openssl.Dockerfile b/.github/workflows/manylinux_2_28-openssl.Dockerfile index 938cbafd37..52931c4334 100644 --- a/.github/workflows/manylinux_2_28-openssl.Dockerfile +++ b/.github/workflows/manylinux_2_28-openssl.Dockerfile @@ -1,5 +1,5 @@ ARG CPU_ARCH=x86_64 -FROM quay.io/pypa/manylinux_2_28_$CPU_ARCH +FROM $BASE_IMAGE ARG OPENSSL_VERSION LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 014d4e45e1..0e390b315c 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -14,6 +14,7 @@ jobs: main: env: REGISTRY: ghcr.io + CIBW_PINNED_DOCKER_IMAGE_CONFIG_PATH: cibuildwheel/resources/pinned_docker_images.cfg strategy: matrix: arch-and-runner-os: [ @@ -28,9 +29,7 @@ jobs: with: sparse-checkout: | .github/workflows - - # TODO: use same version as in pinned cibuildwheel version - - run: docker pull quay.io/pypa/${{ env.MANYLINUX_TAG }}_${{ matrix.arch-and-runner-os[0] }} + path: aerospike-client-python - uses: docker/login-action@v3 with: @@ -48,15 +47,49 @@ jobs: - name: Set up Docker Buildx so we can cache our Docker image layers uses: docker/setup-buildx-action@v3 + - name: Get cibuildwheel version used to build wheels + id: get_cibw_version + run: cibw_version=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_OUTPUT + + - uses: actions/checkout@v4 + with: + repository: pypa/cibuildwheel + ref: ${{ steps.get_cibw_version.outputs.cibw_version }} + sparse-checkout: | + ${{ env.CIBW_PINNED_DOCKER_IMAGE_CONFIG_PATH }} + sparse-checkout-cone-mode: false + path: cibuildwheel + + - name: Get manylinux base image to use + id: get-manylinux-base-image + run: | + import os + import configparser + + config = configparser.ConfigParser() + config.read('${{ env.CIBW_PINNED_DOCKER_IMAGE_CONFIG_PATH }}') + BASE_IMAGE=config['${{ matrix.arch-and-runner-os[0] }}']['${{ env.MANYLINUX_TAG }}'] + + output_file = os.getenv('GITHUB_OUTPUT') + if not output_file: + print(f"Error: GITHUB_OUTPUT environment variable not found.") + exit(1) + + with open(output_file, 'a') as f: + f.write(f"manylinux_base_image={BASE_IMAGE}\n") + shell: python + - name: Build and push uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again - context: .github/workflows - file: .github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile + context: ${{ github.workspace }}/aerospike-client-python/.github/workflows + file: ${{ github.workspace }}/aerospike-client-python/.github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile build-args: | OPENSSL_VERSION=${{ inputs.openssl-version }} CPU_ARCH=${{ matrix.arch-and-runner-os[0] }} + BASE_IMAGE=${{ steps.get-manylinux-base-image.outputs.manylinux_base_image }} + pull: true # setup-buildx-action configures Docker to use the docker-container build driver # This driver doesn't publish an image locally by default # so we have to manually enable it From 487ed5989b773744685a86101de4c4efdcf3d760 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:22:27 -0700 Subject: [PATCH 2/9] fix wrong dir --- .github/workflows/update-manylinux-openssl-image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 0e390b315c..4347b919a6 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -50,6 +50,7 @@ jobs: - name: Get cibuildwheel version used to build wheels id: get_cibw_version run: cibw_version=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_OUTPUT + working-directory: .github/workflows - uses: actions/checkout@v4 with: @@ -68,7 +69,7 @@ jobs: config = configparser.ConfigParser() config.read('${{ env.CIBW_PINNED_DOCKER_IMAGE_CONFIG_PATH }}') - BASE_IMAGE=config['${{ matrix.arch-and-runner-os[0] }}']['${{ env.MANYLINUX_TAG }}'] + BASE_IMAGE = config['${{ matrix.arch-and-runner-os[0] }}']['${{ env.MANYLINUX_TAG }}'] output_file = os.getenv('GITHUB_OUTPUT') if not output_file: From c0f98a88cc9d8f5dd5ad6fb46482f2fa482eb4e0 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:24:10 -0700 Subject: [PATCH 3/9] fix? --- .github/workflows/update-manylinux-openssl-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 4347b919a6..2f79ae82f3 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -50,7 +50,7 @@ jobs: - name: Get cibuildwheel version used to build wheels id: get_cibw_version run: cibw_version=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_OUTPUT - working-directory: .github/workflows + working-directory: aerospike-client-python/.github/workflows - uses: actions/checkout@v4 with: From dbf81df95d2606d74037eac23ef9c9006342d3e4 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:26:41 -0700 Subject: [PATCH 4/9] github outputs dont show up in logs --- .github/workflows/update-manylinux-openssl-image.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 2f79ae82f3..abbcce2da7 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -48,14 +48,13 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Get cibuildwheel version used to build wheels - id: get_cibw_version - run: cibw_version=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_OUTPUT + run: CIBW_VERSION=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_ENV working-directory: aerospike-client-python/.github/workflows - uses: actions/checkout@v4 with: repository: pypa/cibuildwheel - ref: ${{ steps.get_cibw_version.outputs.cibw_version }} + ref: ${{ env.CIBW_VERSION }} sparse-checkout: | ${{ env.CIBW_PINNED_DOCKER_IMAGE_CONFIG_PATH }} sparse-checkout-cone-mode: false From 1f8705b080ef6e548c2809c271dcfe8e7a647e2f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:27:32 -0700 Subject: [PATCH 5/9] fix --- .github/workflows/update-manylinux-openssl-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index abbcce2da7..6bc84e58e0 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -48,7 +48,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Get cibuildwheel version used to build wheels - run: CIBW_VERSION=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_ENV + run: echo CIBW_VERSION=$(yq eval '.jobs.cibuildwheel.steps | map(select(.id == "cibuildwheel"))[0].uses' build-wheels.yml | cut -f 2- -d "@") >> $GITHUB_ENV working-directory: aerospike-client-python/.github/workflows - uses: actions/checkout@v4 From 53f18a4fb00bf83943a3eca982e2e06271bf964e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:29:27 -0700 Subject: [PATCH 6/9] fix --- .github/workflows/update-manylinux-openssl-image.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 6bc84e58e0..00a95b3571 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -78,13 +78,15 @@ jobs: with open(output_file, 'a') as f: f.write(f"manylinux_base_image={BASE_IMAGE}\n") shell: python + working-directory: cibuildwheel - name: Build and push uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again - context: ${{ github.workspace }}/aerospike-client-python/.github/workflows - file: ${{ github.workspace }}/aerospike-client-python/.github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile + # TODO: not pretty + context: ${{ github.workspace }}/aerospike-client-python/aerospike-client-python/.github/workflows + file: ${{ github.workspace }}/aerospike-client-python/aerospike-client-python/.github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile build-args: | OPENSSL_VERSION=${{ inputs.openssl-version }} CPU_ARCH=${{ matrix.arch-and-runner-os[0] }} From 71b17a35dea9fe412f4549145f42ef0c42818584 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:30:17 -0700 Subject: [PATCH 7/9] fix --- .github/workflows/update-manylinux-openssl-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index 00a95b3571..c770637a89 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -84,9 +84,9 @@ jobs: uses: docker/build-push-action@v6 with: # Don't want to use default Git context or else it will clone the whole Python client repo again - # TODO: not pretty - context: ${{ github.workspace }}/aerospike-client-python/aerospike-client-python/.github/workflows - file: ${{ github.workspace }}/aerospike-client-python/aerospike-client-python/.github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile + # TODO: review how main repo is cloned + context: ${{ github.workspace }}/aerospike-client-python/.github/workflows + file: ${{ github.workspace }}/aerospike-client-python/.github/workflows/${{ env.MANYLINUX_TAG }}-openssl.Dockerfile build-args: | OPENSSL_VERSION=${{ inputs.openssl-version }} CPU_ARCH=${{ matrix.arch-and-runner-os[0] }} From ab0025bff6459515c9817bcc19d6aa965204f3b9 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:34:02 -0700 Subject: [PATCH 8/9] check if we can retrieve base image or its not writing properly to output --- .github/workflows/update-manylinux-openssl-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-manylinux-openssl-image.yml b/.github/workflows/update-manylinux-openssl-image.yml index c770637a89..3bbcfcffcd 100644 --- a/.github/workflows/update-manylinux-openssl-image.yml +++ b/.github/workflows/update-manylinux-openssl-image.yml @@ -76,6 +76,7 @@ jobs: exit(1) with open(output_file, 'a') as f: + print("base image:", BASE_IMAGE) f.write(f"manylinux_base_image={BASE_IMAGE}\n") shell: python working-directory: cibuildwheel From 28a418250386c0e9b359646081482577853c9a02 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:35:55 -0700 Subject: [PATCH 9/9] declare BASE IMAGE as arg --- .github/workflows/manylinux_2_28-openssl.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manylinux_2_28-openssl.Dockerfile b/.github/workflows/manylinux_2_28-openssl.Dockerfile index 52931c4334..5d0bde7af6 100644 --- a/.github/workflows/manylinux_2_28-openssl.Dockerfile +++ b/.github/workflows/manylinux_2_28-openssl.Dockerfile @@ -1,4 +1,5 @@ ARG CPU_ARCH=x86_64 +ARG BASE_IMAGE FROM $BASE_IMAGE ARG OPENSSL_VERSION LABEL com.aerospike.clients.openssl-version=$OPENSSL_VERSION