Merge pull request #2706 from VWS-Python/rm-codecov

Replace codecov with combined report

Author: adamtheturtle

.github/workflows/skip-tests.yml

@@ -1,6 +1,5 @@
 ---
-
-name: CI
+name: Test
 
 on:
   push:
@@ -14,15 +13,13 @@ on:
   workflow_dispatch: {}
 
 # We share Vuforia credentials and therefore Vuforia databases across
-# workflows.
-# We therefore want to run only one workflow at a time.
+# workflows. We therefore want to run only one workflow at a time.
 concurrency: vuforia_credentials
 
 jobs:
-  build:
-
+  # CI tests with matrix
+  ci-tests:
     runs-on: ubuntu-latest
-
     strategy:
       fail-fast: false
       matrix:
@@ -120,9 +117,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v5
-        with:
-          # See https://github.com/codecov/codecov-action/issues/190.
-          fetch-depth: 2
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6
@@ -158,54 +152,189 @@ jobs:
 
       - name: Run tests
         run: |
-          uv run --extra=dev pytest \
-            -s \
-            -vvv \
-            --showlocals \
-            --exitfirst \
-            --cov=src/ \
-            --cov=tests/ \
-            --cov-report=xml \
-            ${{ matrix.ci_pattern }}
+          uv run --extra=dev \
+              coverage run \
+              --parallel-mode \
+              --source=src/ \
+              --source=tests/ \
+              -m pytest \
+                  -s \
+                  -vvv \
+                  --showlocals \
+                  --exitfirst \
+                  ${{ matrix.ci_pattern }}
         env:
           UV_PYTHON: ${{ matrix.python-version }}
 
-      - name: Show coverage file
+      - name: Sanitize pattern for artifact name
+        id: sanitize
+        run: |
+          SANITIZED_PATTERN=$(echo "${{ matrix.ci_pattern }}" | sed 's/::/-/g' | sed 's/:/-/g' | sed 's|/|-|g')
+          echo "name=coverage-data-ci-${{ matrix.python-version }}-${SANITIZED_PATTERN}" >> "$GITHUB_OUTPUT"
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.sanitize.outputs.name }}
+          path: .coverage.*
+          include-hidden-files: true
+          if-no-files-found: error
+
+  # Skip tests
+  skip-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.13']
+        platform: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: '**/pyproject.toml'
+
+      - name: Set secrets file
+        run: |
+          cp ./vuforia_secrets.env.example ./vuforia_secrets.env
+
+      - name: Run tests
         run: |
-          # Sometimes we have been sure that we have 100% coverage, but codecov
-          # says otherwise.
+          uv run --extra=dev \
+            coverage run \
+            --parallel-mode \
+            --source=src/ \
+            --source=tests/ \
+            -m pytest \
+                --skip-docker_build_tests \
+                --skip-docker_in_memory \
+                --skip-mock \
+                --skip-real \
+                --capture=no \
+                -vvv \
+                --exitfirst \
+                .
+        env:
+          UV_PYTHON: ${{ matrix.python-version }}
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-data-skip-tests-${{ matrix.python-version }}
+          path: .coverage.*
+          include-hidden-files: true
+          if-no-files-found: error
+
+  # Windows tests
+  windows-tests:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ['3.13']
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: '**/pyproject.toml'
+
+      - name: Set secrets file
+        run: |
+          cp ./vuforia_secrets.env.example ./vuforia_secrets.env
+
+      - name: Run tests
+        shell: bash
+        run: |
+          # We use pytest-xdist to make this run much faster.
+          # The downside is that we cannot use -s / --capture=no.
+          #
+          # We use coverage to collect coverage data but we currently
+          # do not upload / use it because combining Windows and Linux
+          # coverage is challenging.
           #
-          # We show the coverage file here to help with debugging.
-          # https://github.com/VWS-Python/vws-python-mock/issues/708
-          cat ./coverage.xml
-
-      # We run this job on every PR, on every merge to main, and nightly.
-      # This causes us to hit an issue with Codecov.
-      #
-      # We see "Too many uploads to this commit.".
-      # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574.
-      #
-      # To work around this, we do not upload coverage data on scheduled runs.
-      # We print the event name here to help with debugging.
-      - name: Show event name
+          # We therefore have a few ``# pragma: no cover`` statements.
+          uv run --extra=dev \
+            coverage run \
+            --parallel-mode \
+            --source=src/ \
+            --source=tests/ \
+            -m pytest \
+                --skip-real \
+                -vvv \
+                --exitfirst \
+                -n auto \
+                .
+        env:
+          UV_PYTHON: ${{ matrix.python-version }}
+
+  coverage:
+    name: Combine & check coverage
+    needs: [ci-tests, skip-tests, windows-tests]
+    if: always()
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: '**/pyproject.toml'
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Require 100% Coverage
+        id: coverage
         run: |
-          echo ${{ github.event_name }}
+          uv tool install 'coverage[toml]'
 
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5
+          coverage combine
+          coverage html --skip-covered --skip-empty
+
+          # Report and write to summary.
+          coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY"
+
+          # Report again and fail if under 100%.
+          coverage report --fail-under=100
+
+      - name: Upload HTML report if check failed
+        uses: actions/upload-artifact@v4
         with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
-        if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}
 
-  completion-ci:
-    needs: build
+  # Final completion check
+  completion:
+    needs: [ci-tests, skip-tests, windows-tests, coverage]
     runs-on: ubuntu-latest
-    if: always()  # Run even if one matrix job fails
+    if: always()
     steps:
-      - name: Check matrix job status
+      - name: Check all jobs status
         run: |-
-          if ! ${{ needs.build.result == 'success' }}; then
-            echo "One or more matrix jobs failed"
+          if ! ${{ needs.ci-tests.result == 'success' }}; then
+            echo "CI tests failed"
+            exit 1
+          fi
+          if ! ${{ needs.skip-tests.result == 'success' }}; then
+            echo "Skip tests failed"
+            exit 1
+          fi
+          if ! ${{ needs.windows-tests.result == 'success' }}; then
+            echo "Windows tests failed"
+            exit 1
+          fi
+          if ! ${{ needs.coverage.result == 'success' }}; then
+            echo "Coverage check failed"
             exit 1
           fi