test-dbt-models #560
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test-dbt-models | |
| # This workflow is not configured to run on PRs, because PRs have their own CI | |
| # process defined by the `build-and-test-dbt` workflow. It runs under two | |
| # circumstances: | |
| # | |
| # 1. When manually triggered by a Data Department team member, in order to | |
| # run specific dbt tests using the `select` or `selector` input (runs all | |
| # data integriy tests by default, if both `select` and `selector` | |
| # are blank) | |
| # | |
| # 2. On the cron schedule set below, which runs data integrity tests | |
| # weekly in order to proactively alert the team of any problems. Since | |
| # the cron schedule cannot pass input variables to the workflow, we | |
| # configure the input variables to fall back to defaults that support | |
| # the scheduled job, namely the data integrity tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| select: | |
| description: > | |
| Optional space-separated list of tests to run (defaults to all | |
| non-iasWorld data tests) | |
| required: false | |
| type: string | |
| selector: | |
| description: > | |
| Optional dbt selector representing tests to run (takes precedence | |
| over the above list of tests if both are present) | |
| required: false | |
| type: string | |
| schedule: | |
| # Every Monday at 11am UTC (6am UTC-5) | |
| - cron: '0 11 * * 1' | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| UV_SYSTEM_PYTHON: 1 | |
| jobs: | |
| test-dbt-models: | |
| runs-on: ubuntu-latest | |
| # These permissions are needed to interact with GitHub's OIDC Token endpoint | |
| # so that we can authenticate with AWS | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Validate and parse input variables | |
| id: parse-inputs | |
| run: | | |
| # Default to running all non-iasWorld data tests | |
| SELECT_OPTION="--selector select_data_test_non_iasworld" | |
| if [[ -n "$SELECTOR" ]]; then | |
| SELECT_OPTION="--selector $SELECTOR" | |
| elif [[ -n "$SELECT" ]]; then | |
| SELECT_OPTION="--select $SELECT" | |
| fi | |
| echo "Setting select option to '$SELECT_OPTION'" | |
| echo "select-option=$SELECT_OPTION" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| env: | |
| SELECT: ${{ inputs.select }} | |
| SELECTOR: ${{ inputs.selector }} | |
| - name: Setup dbt | |
| uses: ./.github/actions/setup_dbt | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| - name: Restore dbt state cache | |
| id: cache | |
| uses: ./.github/actions/restore_dbt_cache | |
| with: | |
| path: ${{ env.PROJECT_DIR }}/${{ env.STATE_DIR }} | |
| key: ${{ env.CACHE_KEY }} | |
| - name: Install Python dependencies | |
| working-directory: ${{ env.PROJECT_DIR }} | |
| shell: bash | |
| run: uv pip install ".[dbt_tests]" | |
| - name: Run tests | |
| run: | | |
| DEFER_OPTION="" | |
| if [[ "$CACHE_HIT" == 'true' ]]; then | |
| DEFER_OPTION="--defer --state $STATE_DIR" | |
| fi | |
| # shellcheck disable=SC2086 | |
| dbt test --target "$TARGET" \ | |
| $SELECT_OPTION \ | |
| $DEFER_OPTION | |
| working-directory: ${{ env.PROJECT_DIR }} | |
| shell: bash | |
| env: | |
| CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} | |
| SELECT_OPTION: ${{ steps.parse-inputs.outputs.select-option }} | |
| - name: Get current time | |
| if: failure() | |
| run: echo "TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%S")" >> "$GITHUB_ENV" | |
| shell: bash | |
| # Only triggered when run on a schedule. Otherwise, whoever dispatched the | |
| # workflow is notified via GitHub (instead of SNS) | |
| - name: Send failure notification | |
| if: github.event_name == 'schedule' && failure() | |
| uses: ./.github/actions/publish_sns_topic | |
| with: | |
| sns_topic_arn: ${{ secrets.AWS_SNS_NOTIFICATION_TOPIC_ARN }} | |
| subject: "dbt tests failed for workflow run: ${{ github.run_id }}" | |
| body: | | |
| dbt tests failed for workflow ${{ github.run_id }}, run on ${{ env.TIMESTAMP }} UTC | |
| Link to failing workflow: | |
| https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} |