Skip to content

Commit e03d881

Browse files
bobbai00bobbai00
committed
add nightly build
1 parent 4f75e3b commit e03d881

File tree

1 file changed

+71
-28
lines changed

1 file changed

+71
-28
lines changed

‎.github/workflows/build-and-push-images.yml‎

Lines changed: 71 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -54,17 +54,54 @@ on:
5454
required: false
5555
default: false
5656
type: boolean
57+
schedule:
58+
# Run nightly at 2:00 AM UTC
59+
- cron: '0 2 * * *'
5760

5861
concurrency:
59-
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.image_tag }}
62+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.image_tag || 'nightly' }}
6063
cancel-in-progress: false
6164

6265
env:
63-
DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }} # Docker registry namespace
66+
DOCKER_REGISTRY: apache # Will be overridden by job-level env if needed
6467

6568
jobs:
69+
# Step 0: Set runtime parameters (handles both manual and scheduled runs)
70+
set-parameters:
71+
runs-on: ubuntu-latest
72+
outputs:
73+
branch: ${{ steps.set-params.outputs.branch }}
74+
image_tag: ${{ steps.set-params.outputs.image_tag }}
75+
docker_registry: ${{ steps.set-params.outputs.docker_registry }}
76+
services: ${{ steps.set-params.outputs.services }}
77+
platforms: ${{ steps.set-params.outputs.platforms }}
78+
with_r_support: ${{ steps.set-params.outputs.with_r_support }}
79+
steps:
80+
- name: Set build parameters
81+
id: set-params
82+
run: |
83+
# Detect if this is a scheduled run
84+
if [[ "${{ github.event_name }}" == "schedule" ]]; then
85+
echo "Nightly build detected - using nightly defaults"
86+
echo "branch=main" >> $GITHUB_OUTPUT
87+
echo "image_tag=nightly" >> $GITHUB_OUTPUT
88+
echo "docker_registry=apache" >> $GITHUB_OUTPUT
89+
echo "services=*" >> $GITHUB_OUTPUT
90+
echo "platforms=both" >> $GITHUB_OUTPUT
91+
echo "with_r_support=false" >> $GITHUB_OUTPUT
92+
else
93+
echo "Manual workflow_dispatch - using user inputs"
94+
echo "branch=${{ github.event.inputs.branch || 'main' }}" >> $GITHUB_OUTPUT
95+
echo "image_tag=${{ github.event.inputs.image_tag }}" >> $GITHUB_OUTPUT
96+
echo "docker_registry=${{ github.event.inputs.docker_registry || 'apache' }}" >> $GITHUB_OUTPUT
97+
echo "services=${{ github.event.inputs.services || '*' }}" >> $GITHUB_OUTPUT
98+
echo "platforms=${{ github.event.inputs.platforms || 'both' }}" >> $GITHUB_OUTPUT
99+
echo "with_r_support=${{ github.event.inputs.with_r_support || 'false' }}" >> $GITHUB_OUTPUT
100+
fi
101+
66102
# Step 1: Generate JOOQ code once and share it
67103
generate-jooq:
104+
needs: [set-parameters]
68105
runs-on: ubuntu-latest
69106
env:
70107
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
@@ -74,7 +111,7 @@ jobs:
74111
- name: Checkout Texera
75112
uses: actions/checkout@v5
76113
with:
77-
ref: ${{ github.event.inputs.branch }}
114+
ref: ${{ needs.set-parameters.outputs.branch }}
78115

79116
- name: Setup JDK
80117
uses: actions/setup-java@v5
@@ -122,6 +159,7 @@ jobs:
122159

123160
# Step 2: Parse services and prepare build matrix
124161
prepare-matrix:
162+
needs: [set-parameters]
125163
runs-on: ubuntu-latest
126164
outputs:
127165
matrix: ${{ steps.set-matrix.outputs.matrix }}
@@ -132,12 +170,12 @@ jobs:
132170
- name: Checkout Texera
133171
uses: actions/checkout@v5
134172
with:
135-
ref: ${{ github.event.inputs.branch }}
173+
ref: ${{ needs.set-parameters.outputs.branch }}
136174

137175
- name: Set target platforms
138176
id: set-platforms
139177
run: |
140-
PLATFORM_INPUT="${{ github.event.inputs.platforms }}"
178+
PLATFORM_INPUT="${{ needs.set-parameters.outputs.platforms }}"
141179
142180
case "$PLATFORM_INPUT" in
143181
both)
@@ -163,7 +201,7 @@ jobs:
163201
- name: Discover and parse services
164202
id: set-matrix
165203
run: |
166-
SERVICES="${{ github.event.inputs.services }}"
204+
SERVICES="${{ needs.set-parameters.outputs.services }}"
167205
168206
# Discover all Dockerfiles in bin/ directory
169207
echo "Discovering services from Dockerfiles..."
@@ -257,21 +295,22 @@ jobs:
257295
# Step 3a: Build AMD64 images (runs in parallel with ARM64)
258296
build-amd64:
259297
runs-on: ubuntu-latest
260-
needs: [generate-jooq, prepare-matrix]
298+
needs: [set-parameters, generate-jooq, prepare-matrix]
261299
if: needs.prepare-matrix.outputs.build_amd64 == 'true'
262300
strategy:
263301
matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
264302
fail-fast: false
265303
max-parallel: 8 # Higher parallelism for native builds
266304
env:
305+
DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }}
267306
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
268307
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
269308

270309
steps:
271310
- name: Checkout Texera
272311
uses: actions/checkout@v5
273312
with:
274-
ref: ${{ github.event.inputs.branch }}
313+
ref: ${{ needs.set-parameters.outputs.branch }}
275314

276315
- name: Setup JDK
277316
uses: actions/setup-java@v5
@@ -316,11 +355,11 @@ jobs:
316355
file: ${{ matrix.dockerfile }}
317356
platforms: linux/amd64
318357
push: true
319-
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }}-amd64
358+
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64
320359
cache-from: type=gha,scope=${{ matrix.image_name }}-amd64
321360
cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-amd64
322361
build-args: |
323-
${{ matrix.service == 'computing-unit-master' && github.event.inputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
362+
${{ matrix.service == 'computing-unit-master' && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
324363
labels: |
325364
org.opencontainers.image.title=${{ matrix.image_name }}
326365
org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (AMD64)
@@ -329,21 +368,22 @@ jobs:
329368
# Step 3b: Build ARM64 images (runs in parallel with AMD64)
330369
build-arm64:
331370
runs-on: ubuntu-latest
332-
needs: [generate-jooq, prepare-matrix]
371+
needs: [set-parameters, generate-jooq, prepare-matrix]
333372
if: needs.prepare-matrix.outputs.build_arm64 == 'true'
334373
strategy:
335374
matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
336375
fail-fast: false
337376
max-parallel: 4 # Lower for QEMU builds
338377
env:
378+
DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }}
339379
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
340380
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
341381

342382
steps:
343383
- name: Checkout Texera
344384
uses: actions/checkout@v5
345385
with:
346-
ref: ${{ github.event.inputs.branch }}
386+
ref: ${{ needs.set-parameters.outputs.branch }}
347387

348388
- name: Setup JDK
349389
uses: actions/setup-java@v5
@@ -394,11 +434,11 @@ jobs:
394434
file: ${{ matrix.dockerfile }}
395435
platforms: linux/arm64
396436
push: true
397-
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }}-arm64
437+
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64
398438
cache-from: type=gha,scope=${{ matrix.image_name }}-arm64
399439
cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-arm64
400440
build-args: |
401-
${{ matrix.service == 'computing-unit-master' && github.event.inputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
441+
${{ matrix.service == 'computing-unit-master' && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
402442
labels: |
403443
org.opencontainers.image.title=${{ matrix.image_name }}
404444
org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (ARM64)
@@ -407,11 +447,13 @@ jobs:
407447
# Step 4: Create multi-arch manifests (only if building both platforms)
408448
create-manifests:
409449
runs-on: ubuntu-latest
410-
needs: [prepare-matrix, build-amd64, build-arm64]
450+
needs: [set-parameters, prepare-matrix, build-amd64, build-arm64]
411451
if: always() && needs.prepare-matrix.outputs.need_manifest == 'true'
412452
strategy:
413453
matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
414454
fail-fast: false
455+
env:
456+
DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }}
415457

416458
steps:
417459
- name: Log in to Docker Hub
@@ -424,12 +466,12 @@ jobs:
424466
run: |
425467
# Create manifest list combining both architectures
426468
docker buildx imagetools create -t \
427-
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }} \
428-
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }}-amd64 \
429-
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }}-arm64
469+
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }} \
470+
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 \
471+
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64
430472
431473
# Also tag as 'latest' if requested
432-
if [[ "${{ github.event.inputs.image_tag }}" == "latest" ]]; then
474+
if [[ "${{ needs.set-parameters.outputs.image_tag }}" == "latest" ]]; then
433475
docker buildx imagetools create -t \
434476
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:latest \
435477
${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:latest-amd64 \
@@ -438,25 +480,26 @@ jobs:
438480
439481
- name: Inspect multi-arch manifest
440482
run: |
441-
docker buildx imagetools inspect ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ github.event.inputs.image_tag }}
483+
docker buildx imagetools inspect ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}
442484
443485
# Step 5: Summary report
444486
build-summary:
445487
runs-on: ubuntu-latest
446-
needs: [prepare-matrix, build-amd64, build-arm64, create-manifests]
488+
needs: [set-parameters, prepare-matrix, build-amd64, build-arm64, create-manifests]
447489
if: always()
448490
steps:
449491
- name: Generate build summary
450492
run: |
451493
echo "# Texera Multi-Arch Build Complete (Parallel)" >> $GITHUB_STEP_SUMMARY
452494
echo "" >> $GITHUB_STEP_SUMMARY
453495
echo "## Build Configuration" >> $GITHUB_STEP_SUMMARY
454-
echo "- **Branch:** \`${{ github.event.inputs.branch }}\`" >> $GITHUB_STEP_SUMMARY
455-
echo "- **Registry:** \`${{ github.event.inputs.docker_registry }}\`" >> $GITHUB_STEP_SUMMARY
456-
echo "- **Tag:** \`${{ github.event.inputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
457-
echo "- **Services:** ${{ github.event.inputs.services }}" >> $GITHUB_STEP_SUMMARY
458-
echo "- **Platforms:** ${{ github.event.inputs.platforms }}" >> $GITHUB_STEP_SUMMARY
459-
echo "- **R Support:** ${{ github.event.inputs.with_r_support }}" >> $GITHUB_STEP_SUMMARY
496+
echo "- **Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
497+
echo "- **Branch:** \`${{ needs.set-parameters.outputs.branch }}\`" >> $GITHUB_STEP_SUMMARY
498+
echo "- **Registry:** \`${{ needs.set-parameters.outputs.docker_registry }}\`" >> $GITHUB_STEP_SUMMARY
499+
echo "- **Tag:** \`${{ needs.set-parameters.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
500+
echo "- **Services:** ${{ needs.set-parameters.outputs.services }}" >> $GITHUB_STEP_SUMMARY
501+
echo "- **Platforms:** ${{ needs.set-parameters.outputs.platforms }}" >> $GITHUB_STEP_SUMMARY
502+
echo "- **R Support:** ${{ needs.set-parameters.outputs.with_r_support }}" >> $GITHUB_STEP_SUMMARY
460503
echo "" >> $GITHUB_STEP_SUMMARY
461504
echo "## Build Method" >> $GITHUB_STEP_SUMMARY
462505
echo "**Parallel platform builds** (faster)" >> $GITHUB_STEP_SUMMARY
@@ -469,7 +512,7 @@ jobs:
469512
echo "## Images Published" >> $GITHUB_STEP_SUMMARY
470513
echo "All images are now available as multi-arch manifests at:" >> $GITHUB_STEP_SUMMARY
471514
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
472-
echo "docker pull ${{ github.event.inputs.docker_registry }}/<service-name>:${{ github.event.inputs.image_tag }}" >> $GITHUB_STEP_SUMMARY
515+
echo "docker pull ${{ needs.set-parameters.outputs.docker_registry }}/<service-name>:${{ needs.set-parameters.outputs.image_tag }}" >> $GITHUB_STEP_SUMMARY
473516
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
474517
echo "" >> $GITHUB_STEP_SUMMARY
475518
echo "### Usage" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)