CI #1743
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| pull_request: | |
| schedule: | |
| # Run daily at 01:34 so we get notified if CI is broken before a pull request | |
| # is submitted. | |
| - cron: "34 1 * * *" | |
| permissions: | |
| contents: read | |
| jobs: | |
| go-lint: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| id: setup-go | |
| uses: ./.github/actions/setup-go | |
| with: | |
| cache-prefix: go-lint | |
| - name: Compute tools cache info | |
| id: tools-cache-info | |
| run: | | |
| echo path=$(go env GOPATH)/bin >> $GITHUB_OUTPUT | |
| echo make-hash=$(make -n install-tools | sha256sum | cut -d' ' -f1) >> $GITHUB_OUTPUT | |
| - name: Setup tools cache | |
| uses: actions/cache@v4 | |
| id: tools-cache | |
| with: | |
| path: ${{ steps.tools-cache-info.outputs.path }} | |
| key: tools-go-${{ steps.setup-go.outputs.go-version }}-make-${{ steps.tools-cache-info.outputs.make-hash }} | |
| - name: Install tools | |
| if: steps.tools-cache.outputs.cache-hit != 'true' | |
| env: | |
| GOCACHE: /tmp/tools/go-build | |
| GOMODCACHE: /tmp/tools/go-mod | |
| run: make install-tools | |
| - name: Check formatting | |
| run: | | |
| make go-format | |
| modified=$(git ls-files --modified -- '*.go') | |
| if [ -n "$modified" ]; then | |
| for file in $modified; do | |
| echo "::error file=$file::$file is not formatted properly (hint: run \"make go-format\" to fix this)" | |
| done | |
| exit 1 | |
| fi | |
| - name: Check module files | |
| run: | | |
| go mod tidy | |
| modified=$(git ls-files --modified -- go.{mod,sum}) | |
| if [ -n "$modified" ]; then | |
| for file in $modified; do | |
| echo "::error file=$file::$file is not up to date (hint: run \"go mod tidy\" to fix this)" | |
| done | |
| exit 1 | |
| fi | |
| - name: Check mocks | |
| run: | | |
| make mocks-generate | |
| modified=$(git ls-files --modified -- '*/mock_*.go') | |
| if [ -n "$modified" ]; then | |
| for file in $modified; do | |
| echo "::error file=$file::$file is not up to date (hint: run \"make mocks-generate\" to fix this)" | |
| done | |
| exit 1 | |
| fi | |
| - name: Compute golangci-lint cache info | |
| id: golangci-lint-cache-info | |
| run: | | |
| version_regex=" v([0-9]+\.[0-9]+\.[0-9]+) " | |
| [[ "$(golangci-lint version)" =~ $version_regex ]] | |
| echo version=${BASH_REMATCH[1]} >> $GITHUB_OUTPUT | |
| cache_regex="Dir: (.*) | |
| " | |
| [[ "$(golangci-lint cache status)" =~ $cache_regex ]] | |
| echo path=${BASH_REMATCH[1]} >> $GITHUB_OUTPUT | |
| - name: Setup golangci-lint cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.golangci-lint-cache-info.outputs.path }} | |
| key: golangci-lint-${{ steps.golangci-lint-cache-info.outputs.version }}-go-${{ steps.setup-go.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }} | |
| - name: Run golangci-lint | |
| run: make go-lint | |
| python-lint: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Python Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install pipenv | |
| run: pip install pipenv==2022.12.19 | |
| - name: Run Python linters | |
| run: make python-lint | |
| go-unit-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Unit Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: ./.github/actions/setup-go | |
| with: | |
| cache-prefix: go-unit-tests | |
| - name: Run Go Unit Tests | |
| run: make test-go-unit | |
| go-integration-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Integration Tests (${{ matrix.database-backend }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| database-backend: [sqlite, sqlcipher, postgres] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Compute cache info | |
| id: cache-info | |
| run: | | |
| image=$(yq .services.integration-tests.image < tests/integration/docker-compose.yml) | |
| echo go-version=$(skopeo inspect docker://${image} | jq -r .Digest | cut -d: -f2) >> $GITHUB_OUTPUT | |
| - name: Setup cache | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/go-cache/go-build.tar | |
| key: go-integration-tests-${{ matrix.database-backend }}-image-${{ steps.cache-info.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }} | |
| - name: Restore cache | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| run: docker run --rm -v fml-integration-tests_go-cache:/cache -v /tmp/go-cache:/src alpine tar xf /src/go-build.tar -C /cache | |
| - name: Run Integration Tests | |
| run: make container-test | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| FML_DATABASE_BACKEND: ${{ matrix.database-backend }} | |
| FML_SLOW_TESTS_ENABLED: ${{ github.event_name == 'schedule' }} | |
| - name: Save cache | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: docker run --rm -v fml-integration-tests_go-cache:/cache -v /tmp/go-cache:/dst alpine tar cf /dst/go-build.tar -C /cache go-build | |
| go-compatibility-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Compatibility Tests (Mlflow ${{ matrix.mlflow-version }} - ${{ matrix.database-uri}}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| mlflow-version: | |
| - 1.30.0 | |
| - 2.13.0 | |
| database-uri: | |
| - postgres://postgres:postgres@postgres/postgres | |
| - sqlite:///db/mlflow.db | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Compute cache info | |
| id: cache-info | |
| run: | | |
| image=$(yq .services.integration-tests.image < tests/integration/docker-compose.yml) | |
| echo go-version=$(skopeo inspect docker://${image} | jq -r .Digest | cut -d: -f2) >> $GITHUB_OUTPUT | |
| - name: Setup cache | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/go-cache/go-build.tar | |
| key: go-integration-tests-${{ matrix.database-backend }}-image-${{ steps.cache-info.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }} | |
| - name: Restore cache | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| run: docker run --rm -v fml-integration-tests_go-cache:/cache -v /tmp/go-cache:/src alpine tar xf /src/go-build.tar -C /cache | |
| - name: Run Container Tests | |
| run: make container-compatibility-test | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| MLFLOW_VERSION: ${{ matrix.mlflow-version }} | |
| DATABASE_URI: ${{ matrix.database-uri }} | |
| - name: Save cache | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: docker run --rm -v fml-integration-tests_go-cache:/cache -v /tmp/go-cache:/dst alpine tar cf /dst/go-build.tar -C /cache go-build | |
| python-integration-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Python Integration Tests (${{ matrix.test-targets }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| test-targets: [aim, mlflow, fml_client] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| id: setup-go | |
| uses: ./.github/actions/setup-go | |
| with: | |
| cache-prefix: python-integration-tests | |
| - name: Compute cache info | |
| id: cache-info | |
| run: | | |
| echo go-version=$(skopeo inspect docker://golang:${{ steps.setup-go.outputs.go-version }} | jq -r .Digest | cut -d: -f2) >> $GITHUB_OUTPUT | |
| - name: Setup cache | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/go-cache | |
| key: python-integration-tests-image-${{ steps.cache-info.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }} | |
| - name: Run Python integration tests | |
| run: | | |
| if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then | |
| cache=-cache-from=/tmp/go-cache | |
| else | |
| cache=-cache-to=/tmp/go-cache | |
| fi | |
| go run tests/integration/python/main.go $cache -targets ${{ matrix.test-targets }} | |
| build: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Build (${{ matrix.os }}/${{ matrix.arch }}) | |
| strategy: | |
| matrix: | |
| os: [darwin, linux, windows] | |
| arch: [amd64, arm64] | |
| exclude: | |
| - os: windows | |
| arch: arm64 | |
| include: | |
| - os: darwin | |
| runner: macos-latest | |
| - os: linux | |
| runner: ubuntu-latest | |
| - os: windows | |
| runner: ubuntu-latest | |
| fail-fast: true | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: ./.github/actions/setup-go | |
| with: | |
| cache-prefix: build-${{ matrix.os }}-${{ matrix.arch }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install pipenv | |
| run: pip install pipenv==2022.12.19 | |
| - name: Install arm64 cross-compilation toolchain on Linux | |
| if: matrix.os == 'linux' && matrix.arch == 'arm64' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu libc6-dev-arm64-cross | |
| echo CC=aarch64-linux-gnu-gcc >> $GITHUB_ENV | |
| - name: Install Windows cross-compilation toolchain on Linux | |
| if: matrix.os == 'windows' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64-win32 | |
| echo CC=x86_64-w64-mingw32-gcc >> $GITHUB_ENV | |
| - name: Set up Docker Buildx | |
| if: matrix.os == 'linux' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker metadata | |
| if: matrix.os == 'linux' | |
| id: docker-metadata | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: fasttrackml | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| type=edge | |
| - name: Build software distribution | |
| run: make dist | |
| env: | |
| GOOS: ${{ matrix.os }} | |
| GOARCH: ${{ matrix.arch }} | |
| DOCKER_METADATA: ${{ steps.docker-metadata.outputs.json }} | |
| DOCKER_OUTPUT: type=oci,dest=fasttrackml-oci-${{ matrix.arch }}.tar | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fasttrackml-archives-${{ matrix.os }}-${{ matrix.arch }} | |
| path: dist/* | |
| - name: Upload wheels artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fasttrackml-wheels-${{ matrix.os }}-${{ matrix.arch }} | |
| path: wheelhouse/*.whl | |
| - name: Upload Docker artifact | |
| if: matrix.os == 'linux' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fasttrackml-oci-images-${{ matrix.arch }} | |
| path: fasttrackml-oci-*.tar | |
| # Virtual job that can be configured as a required check before a PR can be merged. | |
| # As GitHub considers a check as successful if it is skipped, we need to check its status in | |
| # another workflow (check-required.yml) and create a check there. | |
| all-required-checks-done: | |
| name: All required checks done | |
| needs: | |
| - go-lint | |
| - python-lint | |
| - go-unit-tests | |
| - go-integration-tests | |
| - python-integration-tests | |
| - build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All required checks done" | |
| # Publish any push to a branch or tag to ghcr.io as a convenience | |
| # Actual release to Docker Hub happens in a different workflow | |
| push-ghcr: | |
| name: Push to GitHub Container Registry | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: all-required-checks-done | |
| permissions: | |
| packages: write | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: fasttrackml-oci-images-* | |
| merge-multiple: true | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Compute repo name | |
| id: repo | |
| run: echo name=ghcr.io/$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_OUTPUT | |
| - name: Push to GitHub Container Registry | |
| run: | | |
| repo=${{ steps.repo.outputs.name }} | |
| tags=($(tar -xOf $(ls fasttrackml-oci-*.tar | head -n1) index.json | jq -r '.manifests[].annotations."org.opencontainers.image.ref.name"')) | |
| for image in fasttrackml-oci-*.tar | |
| do | |
| digest=$(tar -xOf $image index.json | jq -r '.manifests[0].digest') | |
| digests+=($digest) | |
| echo "::group::Pushing $image to $repo@$digest" | |
| skopeo copy oci-archive:$image:${tags[0]} docker://$repo@$digest | |
| echo "::endgroup::" | |
| done | |
| echo "::group::Pushing merged manifest to $repo for tags: ${tags[@]}" | |
| docker buildx imagetools create \ | |
| $(printf -- "--tag $repo:%s " ${tags[@]}) \ | |
| $(printf "$repo@%s " ${digests[@]}) | |
| echo "::endgroup::" | |
| release: | |
| name: Release | |
| needs: all-required-checks-done | |
| if: ${{ !github.event.repository.fork && github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') }} | |
| permissions: | |
| actions: write | |
| contents: write | |
| pages: write | |
| id-token: write | |
| secrets: inherit | |
| uses: ./.github/workflows/release.yml |