add dependency-update workflow template (fixes #683)

.gitattributes

@@ -9,6 +9,7 @@ doc/changes/changelog.md  linguist-generated=true
 .github/workflows/check-release-tag.yml     linguist-generated=true
 .github/workflows/checks.yml                linguist-generated=true
 .github/workflows/ci.yml                    linguist-generated=true
+.github/workflows/dependency-update.yml     linguist-generated=true
 .github/workflows/fast-tests.yml            linguist-generated=true
 .github/workflows/gh-pages.yml              linguist-generated=true
 .github/workflows/matrix-*.yml              linguist-generated=true

doc/changes/unreleased.md

@@ -4,6 +4,8 @@
 
 In this major release, several modifications were made to the PTB's workflow templates:
 
+* For automatically resolving vulnerabilities, the `dependency-update.yml` workflow was
+added. For more details, see the [Dependency Update](https://exasol.github.io/python-toolbox/main/user_guide/features/github_workflows/index.html#dependency-update) section.
 * The periodic run which was previously executed in the `ci.yml` has been moved to its
 own `periodic-validation.yml` and will run weekly. This also has been modified to
 run the `slow-checks.yml` so that more complete linting and coverage information is
@@ -15,6 +17,10 @@ it only executes `gh-pages.yml`.
 add custom `fast-tests-extension.yml` and `merge-gate-extension.yml` files. For more
 details, check out the [Workflow Extensions](https://exasol.github.io/python-toolbox/main/user_guide/features/github_workflows/index.html#workflow-extensions) section.
 
+## Features
+
+* #756: Added `dependency-update.yml` to automate resolving vulnerabilities with a generated pull request
+
 ## Bugfix
 
 * #563: Fixed merge-gate to prevent auto-merges from happening when integration tests failed

doc/user_guide/features/github_workflows/index.rst

@@ -69,6 +69,9 @@ Maintained by the PTB
    * - ``fast-tests.yml``
      - Workflow call
      - Executes unit tests.
+   * - ``dependency-update.yml``
+     - Weekly and manual
+     - Audits project dependencies for known vulnerabilities, updates them with Poetry when needed, and creates a pull request if the ``poetry.lock`` was changed.
    * - ``gh-pages.yml``
      - Workflow call
      - Builds the documentation and deploys it to GitHub Pages.
@@ -138,6 +141,19 @@ and is maintained by the PTB and what is project-specific.
 CI Actions
 ----------
 
+.. _dependency_update:
+
+Dependency Update
+^^^^^^^^^^^^^^^^^
+
+The ``dependency-update.yml`` workflow is used to resolve vulnerabilities by updating our project dependencies.
+
+It can be triggered manually and is also scheduled to run weekly.
+
+The workflow first audits dependencies for known vulnerabilities. If vulnerabilities
+are detected, it updates the dependencies using Poetry. When the ``poetry.lock`` is changed,
+then it creates a pull request with the update.
+
 .. _pr_merge_yml:
 
 Merge

exasol/toolbox/templates/github/workflows/dependency-update.yml

@@ -0,0 +1,115 @@
+name: Dependency Update
+
+on:
+  schedule:
+    # Every Monday at 03:00 UTC
+    - cron: "0 3 * * 1"
+  workflow_dispatch:
+
+jobs:
+  dependency-update:
+    name: Dependency Update
+    runs-on: "(( os_version ))"
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Check out Repository
+        id: check-out-repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Fail if not running on the default branch
+        id: check-branch
+        if: github.ref != format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: actions/github-script@v8
+        with:
+          script: |
+            core.setFailed('Not running on the default branch. github.ref is ${{ github.ref }}')
+
+      - name: Set up Python & Poetry Environment
+        id: set-up-python-and-poetry-environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v6
+        with:
+          python-version: "(( minimum_python_version ))"
+          poetry-version: "(( dependency_manager_version ))"
+
+      - name: Audit Dependencies
+        id: audit-dependencies
+        run: |
+          poetry run -- nox -s dependency:audit | tee vulnerabilities.json
+          LENGTH=$(jq 'length' vulnerabilities.json)
+          echo "count=$LENGTH" >> "$GITHUB_OUTPUT"
+
+      - name: Update Dependencies
+        id: update-dependencies
+        if: steps.audit-dependencies.outputs.count > 0
+        run: poetry update
+
+      - name: Check for poetry.lock Changes
+        id: check-for-poetry-lock-changes
+        if: steps.audit-dependencies.outputs.count > 0
+        run: |
+          if git diff --quiet -- poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Configure git
+        id: configure-git
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          git config --global user.email "opensource@exasol.com"
+          git config --global user.name "Automatic Dependency Updater"
+
+      - name: Create branch
+        id: create-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name="dependency-update/$(date "+%Y-%m-%d_%H:%M:%S")"
+          echo "Creating branch $branch_name"
+          git switch -C "$branch_name"
+
+      - name: Commit Changes & Push
+        id: publish-branch
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        run: |
+          branch_name=$(git rev-parse --abbrev-ref HEAD)
+          git add poetry.lock
+          git commit --message "Updated poetry.lock"
+          git push --set-upstream origin "$branch_name"
+
+      - name: Create Pull Request
+        id: create-pr
+        if: steps.check-for-poetry-lock-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          BASE_BRANCH=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name)
+
+          PR_BODY="Automated dependency update for \`poetry.lock\`.
+          This PR was created by the dependency update workflow after running:
+          - \`poetry run -- nox -s dependency:audit\`
+          - \`poetry update\`"
+
+          PR_URL=$(gh pr create \
+                      --base "$BASE_BRANCH" \
+                      --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \
+                      --body "$PR_BODY")
+
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+
+      - name: Report New Pull Request to Slack Channel
+        id: report-pr-slack
+        if: ${{ steps.create-pr.outputs.pr_url }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: '${{ job.status }}'
+          token: '${{ secrets.GITHUB_TOKEN }}'
+          notification_title: 'Dependency update for {repo} created a Pull Request'
+          message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}'
+        env:
+          SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SECURITY_UPDATES_WEBHOOK }}'

exasol/toolbox/util/dependencies/audit.py

@@ -260,7 +260,7 @@ def load_from_pip_audit(cls, working_directory: Path) -> Vulnerabilities:
 
         vulnerabilities = []
         for entry in audit_dict["dependencies"]:
-            for vuln_entry in entry["vulns"]:
+            for vuln_entry in entry.get("vulns", []):
                 vulnerabilities.append(
                     Vulnerability.from_audit_entry(
                         package_name=entry["name"],

test/integration/project-template/nox_test.py

@@ -83,4 +83,4 @@ def test_install_github_workflows(self, poetry_path, run_command):
         assert output.returncode == 0
 
         file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines()
-        assert len(file_list) == 15
+        assert len(file_list) == 16

test/unit/nox/_workflow_test.py

@@ -35,7 +35,7 @@ class TestGenerateWorkflow:
     @staticmethod
     @pytest.mark.parametrize(
         "nox_session_runner_posargs, expected_count",
-        [(ALL, 15), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]],
+        [(ALL, 16), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]],
         indirect=["nox_session_runner_posargs"],
     )
     def test_works_as_expected(

test/unit/util/dependencies/audit_test.py

@@ -240,7 +240,13 @@ class TestVulnerabilities:
     @staticmethod
     def test_with_no_vulnerabilities():
         pip_audit_dict = {
-            "dependencies": [{"name": "alabaster", "version": "0.7.16", "vulns": []}]
+            "dependencies": [
+                {
+                    "name": "exasol-toolbox",
+                    "skip_reason": "Dependency not found on PyPI and could not be audited: exasol-toolbox (7.0.0)",
+                },
+                {"name": "alabaster", "version": "0.7.16", "vulns": []},
+            ]
         }
         pip_audit_json = json.dumps(pip_audit_dict)
 

test/unit/util/workflows/templates_test.py

@@ -12,6 +12,7 @@ def test_get_workflow_templates(project_config):
         "checks",
         "ci",
         "fast-tests",
+        "dependency-update",
         "gh-pages",
         "matrix-all",
         "matrix-exasol",