Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 42 additions & 73 deletions .github/workflows/resolve-build-deps.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
name: Resolve Dependencies and Build Wheels

on:
workflow_dispatch:
pull_request:
branches:
- master
Expand All @@ -26,39 +27,36 @@ env:
# https://reproducible-builds.org/specs/source-date-epoch/
SOURCE_DATE_EPOCH: "1580601600"

jobs:
jobs:
# measure-disk-usage.yml depends on this workflow being triggered and completed,
# so it can wait for the build to calculate dependency sizes.
# The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected.

check-dependency-changes:
name: Check dependency changes
check-should-run:
name: Check if build should run
runs-on: ubuntu-22.04
permissions:
actions: write
contents: read
outputs:
dependency_changed: ${{ steps.dependency-check.outputs.dependency_changed }}
builder_changed: ${{ steps.dependency-check.outputs.builder_changed }}
should_run_build: ${{ steps.dependency-check.outputs.should_run_build }}
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Define diff commits
id: set_sha
run: |
if [ "${{ github.event_name }}" == "pull_request" ]; then
PREV_SHA=${{ github.event.pull_request.base.sha }}
CURR_SHA=${{ github.event.pull_request.head.sha }}
else
PREV_SHA=${{ github.event.before }}
CURR_SHA=${{ github.sha }}
fi

echo "prev_sha=$PREV_SHA" >> $GITHUB_OUTPUT
echo "curr_sha=$CURR_SHA" >> $GITHUB_OUTPUT

echo "Current SHA: $CURR_SHA"
echo "Previous SHA: $PREV_SHA"
if: github.event_name != 'workflow_dispatch'
run: .github/workflows/scripts/resolve_deps_define_diff_commits.sh
env:
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
EVENT_BEFORE: ${{ github.event.before }}

- name: Get changed files
id: changed-files
if: github.event_name != 'workflow_dispatch'
run: |
REPO="${{ github.repository }}"

Expand All @@ -70,50 +68,24 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Check for dependency changes
- name: Check if build should run
id: dependency-check
run: |
FILES_CHANGED="${{ steps.changed-files.outputs.files_changed }}"

cat << EOF > dependency_files.txt
agent_requirements\.in
\.github/workflows/resolve-build-deps\.yaml
\.builders/
EOF

cat <<EOF > builder_files.txt
\.builders/
EOF

DEPENDENCY_CHANGED=$(
echo "$FILES_CHANGED" | \
grep -qf dependency_files.txt \
&& echo "true" || echo "false"
)

BUILDER_CHANGED=$(
echo "$FILES_CHANGED" | \
grep -qf builder_files.txt \
&& echo "true" || echo "false"
)


echo "dependency_changed=$DEPENDENCY_CHANGED" | tee -a $GITHUB_OUTPUT
echo "builder_changed=$BUILDER_CHANGED" | tee -a $GITHUB_OUTPUT

run: .github/workflows/scripts/resolve_deps_check_should_run.sh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }}

test:
name: Run tests
needs:
- check-dependency-changes
if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
- check-should-run
if: needs.check-should-run.outputs.should_run_build == 'true'
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Python ${{ env.PYTHON_VERSION }}

- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.PYTHON_VERSION }}
Expand All @@ -129,8 +101,8 @@ jobs:
build:
name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
needs:
- check-dependency-changes
if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
- check-should-run
if: needs.check-should-run.outputs.should_run_build == 'true'
runs-on: ${{ matrix.job.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -161,8 +133,7 @@ jobs:
python-version: ${{ env.PYTHON_VERSION }}

- name: Install management dependencies
run: |
pip install -r .builders/deps/host_dependencies.txt
run: pip install -r .builders/deps/host_dependencies.txt

- name: Log in to GitHub Packages
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
Expand All @@ -172,29 +143,28 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build image and wheels
if: needs.check-dependency-changes.outputs.builder_changed == 'true'
run: |-
python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
if: needs.check-should-run.outputs.builder_changed == 'true'
run: python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3

- name: Pull image and build wheels
if: needs.check-dependency-changes.outputs.builder_changed == 'false'
run: |-
if: needs.check-should-run.outputs.builder_changed == 'false'
run: |
digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest

- name: Publish image
if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}

- name: Save new image digest
if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
run: >-
${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
| cut -d '@' -f 2
> ${{ env.OUT_DIR }}/image_digest

- name: Persist current image digest
if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'false'
if: needs.check-should-run.outputs.builder_changed == 'false'
run: >-
jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
> ${{ env.OUT_DIR }}/image_digest
Expand All @@ -208,8 +178,8 @@ jobs:
build-macos:
name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }}
needs:
- check-dependency-changes
if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
- check-should-run
if: needs.check-should-run.outputs.should_run_build == 'true'
runs-on: ${{ matrix.job.os }}
strategy:
fail-fast: false
Expand All @@ -229,7 +199,7 @@ jobs:

steps:
- name: Set up environment
run: |-
run: |
# We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
# comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
brew remove --force --ignore-dependencies $(brew list --formula)
Expand All @@ -239,7 +209,7 @@ jobs:
env:
# Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.9/python-3.13.9-macos11.pkg"
run: |-
run: |
curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
sudo installer -pkg python3.pkg -target /

Expand All @@ -255,16 +225,15 @@ jobs:
id: cache-builder-root
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
with:
path: |
~/builder_root
path: ~/builder_root
key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }}

- name: Run the build
env:
# This sets the minimum macOS version compatible for all built artifacts
MACOSX_DEPLOYMENT_TARGET: "11.0" # https://docs.datadoghq.com/agent/supported_platforms/?tab=macos
CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
run: |-
run: |
# If we hit the cache, we can skip the builder setup
if [[ ${CACHE_HIT} == "true" ]]; then
${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup
Expand All @@ -282,11 +251,11 @@ jobs:

publish:
name: Publish artifacts and update lockfiles via PR
if: needs.check-dependency-changes.outputs.dependency_changed == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))))
if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))))
needs:
- build
- build-macos
- check-dependency-changes
- check-should-run
runs-on: ubuntu-latest

permissions:
Expand Down Expand Up @@ -328,7 +297,7 @@ jobs:
run: python .builders/lock.py targets

- name: Clean up repository
run: |-
run: |
rm ${{ steps.auth.outputs.credentials_file_path }}
rm -rf targets

Expand Down
32 changes: 32 additions & 0 deletions .github/workflows/scripts/resolve_deps_check_should_run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]; then
builder_changed="false"
should_run_build="true"
else

cat << EOF > dependency_files.txt
agent_requirements\.in
\.github/workflows/resolve-build-deps\.yaml
\.builders/
EOF

cat <<EOF > builder_files.txt
\.builders/
EOF

should_run_build=$(
echo "$FILES_CHANGED" | \
grep -qf dependency_files.txt \
&& echo "true" || echo "false"
)

builder_changed=$(
echo "$FILES_CHANGED" | \
grep -qf builder_files.txt \
&& echo "true" || echo "false"
)
fi

echo "should_run_build=$should_run_build" | tee -a $GITHUB_OUTPUT
echo "builder_changed=$builder_changed" | tee -a $GITHUB_OUTPUT
15 changes: 15 additions & 0 deletions .github/workflows/scripts/resolve_deps_define_diff_commits.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash

if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
prev_sha=$PR_BASE_SHA
curr_sha=$PR_HEAD_SHA
else
prev_sha=$EVENT_BEFORE
curr_sha=$GITHUB_SHA
fi

echo "prev_sha=$prev_sha" >> $GITHUB_OUTPUT
echo "curr_sha=$curr_sha" >> $GITHUB_OUTPUT

echo "Current SHA: $curr_sha"
echo "Previous SHA: $prev_sha"
Loading