@@ -19,8 +19,7 @@ name: Build
1919# The image builds are split per image so they run on separate runners:
2020#
2121# compile --+--> images-cpp (matrix: cache-server, cache-test-client)
22- # +--> python-package (make python-package) -> dacs-wheel artifact
23- # +--> rust-package (make rust-package) -> dacs-rust-crate artifact
22+ # +--> packages (Python wheel and Rust crate)
2423# images-go (matrix: scheduler, controller) # no needs: runs alongside compile
2524#
2625# The two Go images compile Go from source inside their own Dockerfiles and do
5251permissions : {}
5352
5453jobs :
55- # Compile once. Publishes the build tree so images, python-package and the
54+ # Compile once. Publishes the build tree so images, packages, and the
5655 # caller's test jobs can all reuse it. Builds and gha-caches the native-build
5756 # toolchain image so downstream jobs restore it cheaply.
5857 compile :
@@ -106,34 +105,27 @@ jobs:
106105 # private ACR. The tag matches what bldenv.sh looks up (derived from the
107106 # runner arch, so this is correct on amd64 and arm64 runners alike), so
108107 # `make build` reuses this locally loaded image instead of trying to pull
109- # it. We docker save it to an artifact so the downstream in-container jobs
110- # (test, pytest, wheel) can docker load the exact same image instead of
111- # rebuilding it; the gha cache keeps this build step itself fast on re-runs.
108+ # it. BuildKit writes the Docker archive directly so downstream jobs can
109+ # load the exact same image without a separate docker save export.
112110 - name : Resolve native-build image tag
113111 id : image
114112 run : |
115113 arch=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
116114 echo "tag=dacstest.azurecr.io/dacs/native-build:linux-${arch}" >> "$GITHUB_OUTPUT"
117115
118- - name : Build native-build container image
116+ - name : Build and export native-build container image
119117 uses : docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
120118 with :
121119 context : src/images/native-build
122120 file : src/images/native-build/Dockerfile
123121 tags : ${{ steps.image.outputs.tag }}
124- load : true
125122 push : false # build locally only; never pushed to a registry
126123 cache-from : type=gha,scope=native-build-${{ inputs.build_arch }}
127- cache-to : type=gha,scope=native-build-${{ inputs.build_arch }},mode=max
124+ cache-to : type=gha,scope=native-build-${{ inputs.build_arch }},mode=min
125+ outputs : type=docker,dest=/tmp/dacs-native-build-image-${{ inputs.build_arch }}.tar
128126
129- # Publish the toolchain image so the in-container test/pytest/wheel jobs can
130- # load the identical image (docker save -> artifact -> docker load) rather
131- # than reconstructing it from the layer cache in every job.
132- - name : Save native-build image to a tarball
133- env :
134- BUILD_ARCH : ${{ inputs.build_arch }}
135- IMAGE_TAG : ${{ steps.image.outputs.tag }}
136- run : docker save -o "/tmp/dacs-native-build-image-${BUILD_ARCH}.tar" "$IMAGE_TAG"
127+ - name : Load native-build image
128+ run : docker load -i "/tmp/dacs-native-build-image-${BUILD_ARCH}.tar"
137129
138130 - name : Upload native-build image
139131 uses : actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -142,6 +134,7 @@ jobs:
142134 path : /tmp/dacs-native-build-image-${{ inputs.build_arch }}.tar
143135 retention-days : 1
144136 if-no-files-found : error
137+ compression-level : 0
145138
146139 - name : Build
147140 run : make build
@@ -156,9 +149,12 @@ jobs:
156149 # than upload the dirs directly) to preserve the executable bit on the
157150 # compiled binaries, which actions/upload-artifact would otherwise drop.
158151 - name : Pack build tree
159- env :
160- BUILD_ARCH : ${{ inputs.build_arch }}
161- run : tar -cf "/tmp/dacs-build-tree-${BUILD_ARCH}.tar" bin $(find build -name 'CTestTestfile.cmake')
152+ run : |
153+ mapfile -d '' -t ctest_files < <(
154+ find build -name 'CTestTestfile.cmake' -print0
155+ )
156+ tar -cf "/tmp/dacs-build-tree-${BUILD_ARCH}.tar" \
157+ bin "${ctest_files[@]}"
162158
163159 - name : Upload build tree
164160 uses : actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -167,15 +163,16 @@ jobs:
167163 path : /tmp/dacs-build-tree-${{ inputs.build_arch }}.tar
168164 retention-days : 1
169165 if-no-files-found : error
166+ compression-level : 0
170167
171168 # Build the two Go component images (scheduler, controller). They compile Go
172169 # from source inside their own Dockerfiles and do NOT need the C++ build tree,
173170 # so this job has no `needs` and runs concurrently with compile. Each image
174171 # builds on its own runner (matrix) via docker/build-push-action with the
175172 # GitHub Actions layer cache (scoped per image and architecture), so the Go
176173 # module download and build layers persist across runs without amd64 and arm64
177- # overwriting each other. The image is loaded into the Docker store and
178- # uploaded as a tarball .
174+ # overwriting each other. BuildKit exports the image directly as the tarball
175+ # uploaded for downstream consumers .
179176 images-go :
180177 name : Go image (${{ matrix.image.name }})
181178 runs-on : ${{ inputs.runner }}
@@ -215,17 +212,23 @@ jobs:
215212 - name : Set up Docker Buildx
216213 uses : docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
217214
215+ - name : Resolve image version
216+ id : image
217+ run : echo "version=$(./.pipelines/scripts/get-version.sh container)" >> "$GITHUB_OUTPUT"
218+
218219 - name : Build ${{ matrix.image.name }} image
219220 uses : docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
220221 with :
221222 context : ${{ matrix.image.context }}
222223 file : ${{ matrix.image.file }}
223- tags : ${{ matrix.image.name }}:local
224- load : true
224+ tags : |
225+ ${{ matrix.image.name }}:local
226+ ${{ matrix.image.name }}:${{ steps.image.outputs.version }}
227+ outputs : type=docker,dest=${{ runner.temp }}/dacs-image-${{ matrix.image.name }}.tar
225228 cache-from : type=gha,scope=${{ matrix.image.name }}-${{ inputs.build_arch }}
226229 cache-to : type=gha,scope=${{ matrix.image.name }}-${{ inputs.build_arch }},mode=max
227230
228- - name : Save and upload ${{ matrix.image.name }} image
231+ - name : Upload ${{ matrix.image.name }} image
229232 uses : $/.github/actions/upload-image
230233 with :
231234 name : ${{ matrix.image.name }}
@@ -250,8 +253,10 @@ jobs:
250253 image :
251254 - name : cache-server
252255 file : CacheServer.Dockerfile
256+ cache-mode : min
253257 - name : cache-test-client
254258 file : CacheClient.Dockerfile
259+ cache-mode : max
255260 steps :
256261 - name : Harden Runner
257262 uses : step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
@@ -282,30 +287,33 @@ jobs:
282287 BUILD_CONFIGURATION : ${{ inputs.build_type }}
283288 BUILD_ARCH : ${{ inputs.build_arch }}
284289
290+ - name : Resolve image version
291+ id : image
292+ run : echo "version=$(./.pipelines/scripts/get-version.sh container)" >> "$GITHUB_OUTPUT"
293+
285294 - name : Build ${{ matrix.image.name }} image
286295 uses : docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
287296 with :
288297 context : Docker
289298 file : Docker/${{ matrix.image.file }}
290- tags : ${{ matrix.image.name }}:local
291- load : true
299+ tags : |
300+ ${{ matrix.image.name }}:local
301+ ${{ matrix.image.name }}:${{ steps.image.outputs.version }}
292302 push : false
303+ outputs : type=docker,dest=${{ runner.temp }}/dacs-image-${{ matrix.image.name }}.tar
293304 cache-from : type=gha,scope=${{ matrix.image.name }}-${{ inputs.build_arch }}
294- cache-to : type=gha,scope=${{ matrix.image.name }}-${{ inputs.build_arch }},mode=max
305+ cache-to : type=gha,scope=${{ matrix.image.name }}-${{ inputs.build_arch }},mode=${{ matrix.image.cache-mode }}
295306
296- - name : Save and upload ${{ matrix.image.name }} image
307+ - name : Upload ${{ matrix.image.name }} image
297308 uses : $/.github/actions/upload-image
298309 with :
299310 name : ${{ matrix.image.name }}
300311 arch : ${{ inputs.build_arch }}
301312
302- # Build the dacs-client wheel from the compiled native library. Runs in the
303- # native-build container (glibc-consistent with the compiled .so) via the
304- # make target, so the composite restores the toolchain image. Runs in parallel
305- # with the image jobs. Catches packaging regressions (missing data files, broken
306- # auditwheel repair) before publish.
307- python-package :
308- name : Python package (wheel)
313+ # Build the Python wheel and Rust crate in one job so the large native-build
314+ # image and build tree are restored only once.
315+ packages :
316+ name : Python and Rust packages
309317 needs : compile
310318 runs-on : ${{ inputs.runner }}
311319 timeout-minutes : 30
@@ -344,38 +352,6 @@ jobs:
344352 retention-days : 1
345353 if-no-files-found : error
346354
347- # Package the Rust client with the matching architecture's native library.
348- # This job only uploads a workflow artifact and has no package publishing
349- # permissions or registry credentials.
350- rust-package :
351- name : Rust package (crate)
352- needs : compile
353- runs-on : ${{ inputs.runner }}
354- timeout-minutes : 20
355- permissions :
356- contents : read
357- env :
358- BUILD_TYPE : ${{ inputs.build_type }}
359- BUILD_ARCH : ${{ inputs.build_arch }}
360- steps :
361- - name : Harden Runner
362- uses : step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
363- with :
364- egress-policy : audit
365-
366- - name : Check out the code
367- uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
368- with :
369- persist-credentials : false
370- fetch-depth : 0
371- fetch-tags : true
372-
373- - name : Restore build tree
374- uses : $/.github/actions/restore-build
375- with :
376- artifact : dacs-build-tree-${{ inputs.build_arch }}
377- native-build-image-artifact : dacs-native-build-image-${{ inputs.build_arch }}
378-
379355 - name : Cache Cargo dependencies and build outputs
380356 uses : actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
381357 with :
0 commit comments