diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index 7239d13dfc115..a4acb672b0355 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -1,6 +1,7 @@ name: Resolve Dependencies and Build Wheels on: + workflow_dispatch: pull_request: branches: - master @@ -26,39 +27,36 @@ env: # https://reproducible-builds.org/specs/source-date-epoch/ SOURCE_DATE_EPOCH: "1580601600" -jobs: +jobs: # measure-disk-usage.yml depends on this workflow being triggered and completed, # so it can wait for the build to calculate dependency sizes. # The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected. - check-dependency-changes: - name: Check dependency changes + check-should-run: + name: Check if build should run runs-on: ubuntu-22.04 permissions: actions: write + contents: read outputs: - dependency_changed: ${{ steps.dependency-check.outputs.dependency_changed }} builder_changed: ${{ steps.dependency-check.outputs.builder_changed }} + should_run_build: ${{ steps.dependency-check.outputs.should_run_build }} steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Define diff commits id: set_sha - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - PREV_SHA=${{ github.event.pull_request.base.sha }} - CURR_SHA=${{ github.event.pull_request.head.sha }} - else - PREV_SHA=${{ github.event.before }} - CURR_SHA=${{ github.sha }} - fi - - echo "prev_sha=$PREV_SHA" >> $GITHUB_OUTPUT - echo "curr_sha=$CURR_SHA" >> $GITHUB_OUTPUT - - echo "Current SHA: $CURR_SHA" - echo "Previous SHA: $PREV_SHA" + if: github.event_name != 'workflow_dispatch' + run: .github/workflows/scripts/resolve_deps_define_diff_commits.sh + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT_BEFORE: ${{ github.event.before }} - name: Get changed files id: changed-files + if: github.event_name != 'workflow_dispatch' run: | REPO="${{ github.repository }}" @@ -70,50 +68,24 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check for dependency changes + - name: Check if build should run id: dependency-check - run: | - FILES_CHANGED="${{ steps.changed-files.outputs.files_changed }}" - - cat << EOF > dependency_files.txt - agent_requirements\.in - \.github/workflows/resolve-build-deps\.yaml - \.builders/ - EOF - - cat < builder_files.txt - \.builders/ - EOF - - DEPENDENCY_CHANGED=$( - echo "$FILES_CHANGED" | \ - grep -qf dependency_files.txt \ - && echo "true" || echo "false" - ) - - BUILDER_CHANGED=$( - echo "$FILES_CHANGED" | \ - grep -qf builder_files.txt \ - && echo "true" || echo "false" - ) - - - echo "dependency_changed=$DEPENDENCY_CHANGED" | tee -a $GITHUB_OUTPUT - echo "builder_changed=$BUILDER_CHANGED" | tee -a $GITHUB_OUTPUT - + run: .github/workflows/scripts/resolve_deps_check_should_run.sh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }} test: name: Run tests needs: - - check-dependency-changes - if: needs.check-dependency-changes.outputs.dependency_changed == 'true' + - check-should-run + if: needs.check-should-run.outputs.should_run_build == 'true' runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ env.PYTHON_VERSION }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -129,8 +101,8 @@ jobs: build: name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }} needs: - - check-dependency-changes - if: needs.check-dependency-changes.outputs.dependency_changed == 'true' + - check-should-run + if: needs.check-should-run.outputs.should_run_build == 'true' runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -161,8 +133,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install management dependencies - run: | - pip install -r .builders/deps/host_dependencies.txt + run: pip install -r .builders/deps/host_dependencies.txt - name: Log in to GitHub Packages uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 @@ -172,29 +143,28 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build image and wheels - if: needs.check-dependency-changes.outputs.builder_changed == 'true' - run: |- - python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 + if: needs.check-should-run.outputs.builder_changed == 'true' + run: python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 - name: Pull image and build wheels - if: needs.check-dependency-changes.outputs.builder_changed == 'false' - run: |- + if: needs.check-should-run.outputs.builder_changed == 'false' + run: | digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json) python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest - name: Publish image - if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true' + if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true' run: ${DOCKER} push ${{ env.BUILDER_IMAGE }} - name: Save new image digest - if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true' + if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true' run: >- ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }} | cut -d '@' -f 2 > ${{ env.OUT_DIR }}/image_digest - name: Persist current image digest - if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'false' + if: needs.check-should-run.outputs.builder_changed == 'false' run: >- jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json > ${{ env.OUT_DIR }}/image_digest @@ -208,8 +178,8 @@ jobs: build-macos: name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }} needs: - - check-dependency-changes - if: needs.check-dependency-changes.outputs.dependency_changed == 'true' + - check-should-run + if: needs.check-should-run.outputs.should_run_build == 'true' runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -229,7 +199,7 @@ jobs: steps: - name: Set up environment - run: |- + run: | # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate` brew remove --force --ignore-dependencies $(brew list --formula) @@ -239,7 +209,7 @@ jobs: env: # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.9/python-3.13.9-macos11.pkg" - run: |- + run: | curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg sudo installer -pkg python3.pkg -target / @@ -255,8 +225,7 @@ jobs: id: cache-builder-root uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - path: | - ~/builder_root + path: ~/builder_root key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }} - name: Run the build @@ -264,7 +233,7 @@ jobs: # This sets the minimum macOS version compatible for all built artifacts MACOSX_DEPLOYMENT_TARGET: "11.0" # https://docs.datadoghq.com/agent/supported_platforms/?tab=macos CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }} - run: |- + run: | # If we hit the cache, we can skip the builder setup if [[ ${CACHE_HIT} == "true" ]]; then ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup @@ -282,11 +251,11 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - if: needs.check-dependency-changes.outputs.dependency_changed == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) + if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) needs: - build - build-macos - - check-dependency-changes + - check-should-run runs-on: ubuntu-latest permissions: @@ -328,7 +297,7 @@ jobs: run: python .builders/lock.py targets - name: Clean up repository - run: |- + run: | rm ${{ steps.auth.outputs.credentials_file_path }} rm -rf targets diff --git a/.github/workflows/scripts/resolve_deps_check_should_run.sh b/.github/workflows/scripts/resolve_deps_check_should_run.sh new file mode 100755 index 0000000000000..b036997b869ab --- /dev/null +++ b/.github/workflows/scripts/resolve_deps_check_should_run.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]; then + builder_changed="false" + should_run_build="true" +else + + cat << EOF > dependency_files.txt +agent_requirements\.in +\.github/workflows/resolve-build-deps\.yaml +\.builders/ +EOF + + cat < builder_files.txt +\.builders/ +EOF + + should_run_build=$( + echo "$FILES_CHANGED" | \ + grep -qf dependency_files.txt \ + && echo "true" || echo "false" + ) + + builder_changed=$( + echo "$FILES_CHANGED" | \ + grep -qf builder_files.txt \ + && echo "true" || echo "false" + ) +fi + +echo "should_run_build=$should_run_build" | tee -a $GITHUB_OUTPUT +echo "builder_changed=$builder_changed" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/resolve_deps_define_diff_commits.sh b/.github/workflows/scripts/resolve_deps_define_diff_commits.sh new file mode 100755 index 0000000000000..2b75dcded1792 --- /dev/null +++ b/.github/workflows/scripts/resolve_deps_define_diff_commits.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + prev_sha=$PR_BASE_SHA + curr_sha=$PR_HEAD_SHA +else + prev_sha=$EVENT_BEFORE + curr_sha=$GITHUB_SHA +fi + +echo "prev_sha=$prev_sha" >> $GITHUB_OUTPUT +echo "curr_sha=$curr_sha" >> $GITHUB_OUTPUT + +echo "Current SHA: $curr_sha" +echo "Previous SHA: $prev_sha"