Skip to content

Commit

Permalink
Apply zizmor findings (#45408)
Browse files Browse the repository at this point in the history
  • Loading branch information
gopidesupavan authored Jan 5, 2025
1 parent 0779b2c commit c5049d0
Show file tree
Hide file tree
Showing 26 changed files with 218 additions and 89 deletions.
3 changes: 1 addition & 2 deletions .github/actions/breeze/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ inputs:
default: "3.9"
use-uv:
description: 'Whether to use uv tool'
required: "true"
type: "string"
required: true
outputs:
host-python-version:
description: Python version used in host
Expand Down
10 changes: 7 additions & 3 deletions .github/actions/install-pre-commit/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,14 @@ runs:
steps:
- name: Install pre-commit, uv, and pre-commit-uv
shell: bash
env:
UV_VERSION: ${{inputs.uv-version}}
PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}}
PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}}
run: |
pip install uv==${{inputs.uv-version}} || true
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
pip install uv==${UV_VERSION} || true
uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \
--with pre-commit-uv==${PRE_COMMIT_UV_VERSION}
working-directory: ${{ github.workspace }}
# We need to use tar file with archive to restore all the permissions and symlinks
- name: "Delete ~.cache"
Expand Down
8 changes: 6 additions & 2 deletions .github/actions/prepare_breeze_and_image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ runs:
key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
path: "/tmp/"
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
env:
PLATFORM: ${{ inputs.platform }}
PYTHON: ${{ inputs.python }}
IMAGE_TYPE: ${{ inputs.image-type }}
run: >
breeze ${{ inputs.image-type }}-image load
--platform ${{ inputs.platform }} --python ${{ inputs.python }}
breeze ${IMAGE_TYPE}-image load
--platform ${PLATFORM} --python ${PYTHON}
shell: bash
5 changes: 4 additions & 1 deletion .github/actions/prepare_single_ci_image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ runs:
path: "/tmp/"
if: contains(inputs.python-versions-list-as-string, inputs.python)
- name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
run: breeze ci-image load --platform "${{ inputs.platform }}" --python "${{ inputs.python }}"
env:
PLATFORM: ${{ inputs.platform }}
PYTHON: ${{ inputs.python }}
run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}"
shell: bash
if: contains(inputs.python-versions-list-as-string, inputs.python)
5 changes: 4 additions & 1 deletion .github/workflows/additional-ci-image-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,10 @@ jobs:
with:
use-uv: ${{ inputs.use-uv }}
- name: "Login to ghcr.io"
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
env:
actor: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin
- name: "Check that image builds quickly"
run: breeze shell --max-time 600 --platform "linux/amd64"

Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/additional-prod-image-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,15 @@ jobs:
python: ${{ inputs.default-python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Test examples of PROD image building"
env:
GITHUB_REPOSITORY: ${{ github.repository }}
DEFAULT_BRANCH: ${{ inputs.default-branch }}
DEFAULT_PYTHON_VERSION: ${{ inputs.default-python-version }}
run: "
cd ./docker_tests && \
python -m pip install -r requirements.txt && \
TEST_IMAGE=\"ghcr.io/${{ github.repository }}/${{ inputs.default-branch }}\
/prod/python${{ inputs.default-python-version }}\" \
TEST_IMAGE=\"ghcr.io/$GITHUB_REPOSITORY/$DEFAULT_BRANCH\
/prod/python$DEFAULT_PYTHON_VERSION\" \
python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes"

test-docker-compose-quick-start:
Expand Down
23 changes: 15 additions & 8 deletions .github/workflows/backport-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,24 +64,28 @@ jobs:
id: execute-backport
env:
GH_AUTH: ${{ secrets.GITHUB_TOKEN }}
TARGET_BRANCH: ${{ inputs.target-branch }}
COMMIT_SHA: ${{ inputs.commit-sha }}
run: |
git config --global user.email "[email protected]"
git config --global user.name "Your Name"
set +e
{
echo 'cherry_picker_output<<EOF'
cherry_picker ${{ inputs.commit-sha }} ${{ inputs.target-branch }}
cherry_picker ${COMMIT_SHA} ${TARGET_BRANCH}
echo EOF
} >> "${GITHUB_OUTPUT}"
continue-on-error: true

- name: Parse backport output
id: parse-backport-output
env:
CHERRY_PICKER_OUTPUT: ${{ steps.execute-backport.outputs.cherry_picker_output }}
run: |
set +e
echo "${{ steps.execute-backport.outputs.cherry_picker_output }}"
echo "${CHERRY_PICKER_OUTPUT}"
url=$(echo "${{ steps.execute-backport.outputs.cherry_picker_output }}" | \
url=$(echo "${CHERRY_PICKER_OUTPUT}" | \
grep -o 'Backport PR created at https://[^ ]*' | \
awk '{print $5}')
Expand All @@ -99,17 +103,20 @@ jobs:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
COMMIT_SHA: ${{ inputs.commit-sha }}
TARGET_BRANCH: ${{ inputs.target-branch }}
BACKPORT_URL: ${{ steps.parse-backport-output.outputs.backport-url }}
run: |
COMMIT_INFO_URL="https://api.github.com/repos/${{ github.repository }}/commits/"
COMMIT_INFO_URL="${COMMIT_INFO_URL}${{ inputs.commit-sha }}/pulls"
COMMIT_INFO_URL="https://api.github.com/repos/$REPOSITORY/commits/"
COMMIT_INFO_URL="${COMMIT_INFO_URL}$COMMIT_SHA/pulls"
PR_NUMBER=$(gh api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/commits/${{ inputs.commit-sha }}/pulls \
/repos/$REPOSITORY/commits/$COMMIT_SHA/pulls \
--jq '.[0].number')
python ./dev/backport/update_backport_status.py \
${{ steps.parse-backport-output.outputs.backport-url }} \
${{ inputs.commit-sha }} ${{ inputs.target-branch }} \
$BACKPORT_URL \
$COMMIT_SHA $TARGET_BRANCH \
"$PR_NUMBER"
20 changes: 15 additions & 5 deletions .github/workflows/ci-image-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -140,17 +140,22 @@ jobs:
path: "/tmp/"
id: restore-cache-mount
- name: "Import mount-cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
run: >
breeze ci-image import-mount-cache
--cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz
--cache-file /tmp/ci-cache-mount-save-v2-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: steps.restore-cache-mount.outputs.stash-hit == 'true'
- name: "Login to ghcr.io"
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTOR: ${{ github.actor }}
run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
- name: >
Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} image
run: >
breeze ci-image build --platform "${{ inputs.platform }}"
breeze ci-image build --platform "${PLATFORM}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
Expand All @@ -167,8 +172,11 @@ jobs:
GITHUB_USERNAME: ${{ github.actor }}
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
PLATFORM: ${{ inputs.platform }}
- name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
run: breeze ci-image save --platform "${{ inputs.platform }}"
env:
PLATFORM: ${{ inputs.platform }}
run: breeze ci-image save --platform "${PLATFORM}"
if: inputs.upload-image-artifact == 'true'
- name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
Expand All @@ -179,9 +187,11 @@ jobs:
retention-days: '2'
if: inputs.upload-image-artifact == 'true'
- name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
run: >
breeze ci-image export-mount-cache
--cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz
--cache-file /tmp/ci-cache-mount-save-v2-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: inputs.upload-mount-cache-artifact == 'true'
- name: "Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
Expand Down
11 changes: 8 additions & 3 deletions .github/workflows/ci-image-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,14 +228,15 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
- name: "MyPy checks for ${{ matrix.mypy-check }}"
run: pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files
run: pre-commit run --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files
env:
VERBOSE: "false"
COLUMNS: "250"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ inputs.branch }}
RUFF_FORMAT: "github"
INCLUDE_MYPY_VOLUME: "false"
MYPY_CHECK: ${{ matrix.mypy-check }}

build-docs:
timeout-minutes: 150
Expand Down Expand Up @@ -276,8 +277,10 @@ jobs:
key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
id: restore-docs-inventory-cache
- name: "Building docs with ${{ matrix.flag }} flag"
env:
DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
run: >
breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }}
breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
- name: "Save docs inventory cache"
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
with:
Expand Down Expand Up @@ -339,9 +342,11 @@ jobs:
python: ${{ inputs.default-python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Publish docs"
env:
DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
run: >
breeze release-management publish-docs --override-versioned --run-in-parallel
${{ inputs.docs-list-as-string }}
${DOCS_LIST_AS_STRING}
- name: Check disk space available
run: df -h
- name: "Generate back references for providers"
Expand Down
9 changes: 3 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,6 @@ on: # yamllint disable-line rule:truthy
permissions:
# All other permissions are set to none by default
contents: read
# Technically read access while waiting for images should be more than enough. However,
# there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission
# denied error when attempting to just pull private image, changing the token permission to write solves the
# issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml
# changes. If it is for a fork, then the token is read-only anyway.
packages: write
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down Expand Up @@ -228,6 +222,9 @@ jobs:
name: "Additional CI image checks"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/additional-ci-image-checks.yml
permissions:
contents: read
packages: write
if: needs.build-info.outputs.canary-run == 'true'
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
Expand Down
12 changes: 9 additions & 3 deletions .github/workflows/generate-constraints.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,22 +99,28 @@ jobs:
# from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
# And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
shell: bash
env:
CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }}
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--package-format wheel --version-suffix-for-pypi dev0
${{ inputs.chicken-egg-providers }}
${CHICKEN_EGG_PROVIDERS}
if: inputs.chicken-egg-providers != ''
- name: "PyPI constraints"
shell: bash
timeout-minutes: 25
env:
CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }}
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints --answer yes
--chicken-egg-providers "${{ inputs.chicken-egg-providers }}" --parallelism 3
--chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" --parallelism 3
- name: "Dependency upgrade summary"
shell: bash
env:
PYTHON_VERSIONS: ${{ env.PYTHON_VERSIONS }}
run: |
for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
for PYTHON_VERSION in $PYTHON_VERSIONS; do
echo "Summarizing Python $PYTHON_VERSION"
cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
done
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/helm-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ jobs:
python: ${{ inputs.default-python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Helm Unit Tests: ${{ matrix.helm-test-package }}"
run: breeze testing helm-tests --test-type "${{ matrix.helm-test-package }}"
env:
HELM_TEST_PACKAGE: "${{ matrix.helm-test-package }}"
run: breeze testing helm-tests --test-type "${HELM_TEST_PACKAGE}"

tests-helm-release:
timeout-minutes: 80
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/integration-system-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,10 @@ jobs:
python: ${{ inputs.default-python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Integration: core ${{ matrix.integration }}"
env:
INTEGRATION: "${{ matrix.integration }}"
# yamllint disable rule:line-length
run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${{ matrix.integration }}"
run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${INTEGRATION}"
- name: "Post Tests success"
uses: ./.github/actions/post_tests_success
with:
Expand Down Expand Up @@ -148,7 +150,9 @@ jobs:
python: ${{ inputs.default-python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Integration: providers ${{ matrix.integration }}"
run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${{ matrix.integration }}"
env:
INTEGRATION: "${{ matrix.integration }}"
run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${INTEGRATION}"
- name: "Post Tests success"
uses: ./.github/actions/post_tests_success
with:
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/k8s-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,11 @@ jobs:
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Prepare PYTHON_MAJOR_MINOR_VERSION and KUBERNETES_VERSION"
id: prepare-versions
env:
KUBERNETES_COMBO: ${{ matrix.kubernetes-combo }}
run: |
echo "PYTHON_MAJOR_MINOR_VERSION=${{ matrix.kubernetes-combo }}" | sed 's/-.*//' >> $GITHUB_ENV
echo "KUBERNETES_VERSION=${{ matrix.kubernetes-combo }}" | sed 's/=[^-]*-/=/' >> $GITHUB_ENV
echo "PYTHON_MAJOR_MINOR_VERSION=${KUBERNETES_COMBO}" | sed 's/-.*//' >> $GITHUB_ENV
echo "KUBERNETES_VERSION=${KUBERNETES_COMBO}" | sed 's/=[^-]*-/=/' >> $GITHUB_ENV
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/news-fragment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@ jobs:
fetch-depth: 0

- name: Check news fragment existence
env:
BASE_REF: ${{ github.base_ref }}
run: >
python -m pip install --upgrade uv &&
uv tool run towncrier check
--dir .
--config newsfragments/config.toml
--compare-with origin/${{ github.base_ref }}
--compare-with origin/${BASE_REF}
||
{
printf "\033[1;33mMissing significant newsfragment for PR labeled with
Expand All @@ -54,6 +56,8 @@ jobs:
; }
- name: Check news fragment contains change types
env:
BASE_REF: ${{ github.base_ref }}
run: >
change_types=(
'DAG changes'
Expand All @@ -64,7 +68,7 @@ jobs:
'Plugin changes'
'Dependency change'
)
news_fragment_content=`git diff origin/${{ github.base_ref }} newsfragments/*.significant.rst`
news_fragment_content=`git diff origin/${BASE_REF} newsfragments/*.significant.rst`
for type in "${change_types[@]}"; do
if [[ $news_fragment_content != *"$type"* ]]; then
Expand Down
Loading

0 comments on commit c5049d0

Please sign in to comment.