python.yaml

name: Python package

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  release:
    types:
      - published

# Newer commits should cancel old runs
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          # dependencies: python.version_min
          python-version: "3.10"
      - name: Install nmhit (provides the nmhit-format CLI used by pre-commit)
        # dependencies: nmhit.version_min
        run: pip install 'nmhit>=0.2.1'
      - name: Run ruff lint
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: ruff --all-files
      - name: Run ruff format check
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: ruff-format --all-files
      - name: Run nmhit-format check
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: nmhit-format --all-files

  typecheck:
    needs: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/neml2-ci
        with:
          # dependencies: python.version_min
          python-version: "3.10"
          # dependencies: cmake.version_min
          cmake-version: "3.26.1"
          # dependencies: torch.version
          torch-version: "2.12.1"
          cpu-torch: true
      - name: Install (editable)
        # Editable install is load-bearing for this job: pyright type-checks
        # the in-tree ``./neml2/`` (per ``[tool.pyright].include``), and
        # ``neml2-stub`` writes ``_aoti.pyi`` next to the .so it discovers.
        # A non-editable install would place both the .so and the generated
        # .pyi under ``site-packages/neml2/aoti/`` -- outside pyright's
        # include list -- so the ``from ._aoti import Model`` line in
        # ``neml2/aoti/__init__.py`` would fail with "could not be
        # resolved". Editable install lets scikit-build-core place the .so
        # into ``./neml2/aoti/`` directly; the stub then lands beside it
        # where pyright sees it.
        run: pip install -e ".[dev]" -v
      - name: Regenerate pybind11 stubs
        # pybind11 extensions ship as .so; pyright can't introspect them
        # without a .pyi sibling. The `neml2-stub` console script (added
        # to [project.scripts]) wraps pybind11-stubgen and writes the
        # stubs next to each .so. Run AFTER `pip install` so the
        # package is fully importable.
        run: neml2-stub
      - name: Run pyright
        run: pyright

  vuln-scan:
    # ``pip-audit`` resolves installed packages against the PyPI advisory
    # database (a mirror of OSV.dev). It flags both direct and transitive
    # dependencies; running it against the installed editable env catches
    # any CVE in the pinned torch, pyzag, sphinx, etc. before merge.
    needs: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/neml2-ci
        with:
          # dependencies: python.version_min
          python-version: "3.10"
          # dependencies: cmake.version_min
          cmake-version: "3.26.1"
          # dependencies: torch.version
          torch-version: "2.12.1"
          cpu-torch: true
      - name: Install
        run: pip install ".[dev]" -v
      - name: Install pip-audit
        run: pip install pip-audit
      - name: Scan installed env for known CVEs
        # ``--vulnerability-service osv`` queries OSV.dev rather than the
        # PyPI advisory mirror; OSV resolves packages installed from non-PyPI
        # indexes (torch's ``+cpu`` / ``+cu126`` local-version specifiers)
        # which the PyPI service rejects with "dependency not found".
        # ``--strict`` exits non-zero on any vulnerability *or* dependency
        # resolution skip; ``--ignore-vuln`` slots in here as needed when a
        # flagged CVE has no realistic exploit path for this codebase.
        run: pip-audit --vulnerability-service osv --strict

  coverage:
    # Python branch + line coverage. Runs the fast lane only -- the
    # regression and verification suites pin gold outputs rather than
    # exercise code paths, so including them would only inflate the
    # coverage number with end-to-end exercises (and triple the CI
    # wall time). The xdist ``-n auto`` matches what the AOTI sweep
    # benchmark showed scales best on a multi-core box; ``parallel =
    # true`` in pyproject.toml's [tool.coverage.run] keeps per-worker
    # .coverage files distinct and merges them cleanly.
    needs: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/neml2-ci
        with:
          # dependencies: python.version_min
          python-version: "3.10"
          # dependencies: cmake.version_min
          cmake-version: "3.26.1"
          # dependencies: torch.version
          torch-version: "2.12.1"
          cpu-torch: true
      - name: Install
        run: pip install ".[dev]" -v
      - name: Run tests with coverage
        run: |
          pytest -n auto --cov --cov-report=xml --cov-report=term \
            --ignore=tests/regression --ignore=tests/verification \
            tests/
      - name: Upload coverage to Codecov
        # Codecov rejects tokenless OIDC uploads on protected branches,
        # so the workflow always passes ``CODECOV_TOKEN`` (a repository
        # secret, generated from the project's Codecov dashboard). For
        # same-repo PRs and pushes to ``main`` the secret resolves and
        # upload proceeds; for fork PRs the secret is unavailable and
        # the action falls back to tokenless OIDC (still allowed for
        # non-protected branches under the org policy).
        uses: codecov/codecov-action@v7
        with:
          files: ./coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false              # Codecov outage shouldn't break PRs
          flags: unittests                     # groups runs in the trend view
      - name: Upload raw coverage as workflow artifact
        # Belt-and-suspenders for the first few weeks after Codecov goes
        # live -- a self-hosted fallback in case the Codecov upload
        # silently breaks. Drop this step once Codecov is reliably
        # picking up uploads.
        uses: actions/upload-artifact@v7
        with:
          name: coverage-xml
          path: coverage.xml
          if-no-files-found: error
          retention-days: 30

  test:
    needs: lint
    strategy:
      fail-fast: false
      matrix:
        runner: [ubuntu-latest, macos-26, gpu_runner]
    runs-on: ${{ matrix.runner }}
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/neml2-ci
        with:
          # dependencies: python.version_min
          python-version: "3.10"
          # dependencies: cmake.version_min
          cmake-version: "3.26.1"
          # dependencies: torch.version
          torch-version: "2.12.1"
          cpu-torch: ${{ matrix.runner != 'gpu_runner' }}
      - name: Install
        run: pip install ".[dev]" -v
      - name: Run tests
        run: pytest -v --junitxml=pytest.xml tests
      - name: Test results
        uses: EnricoMi/publish-unit-test-result-action/linux@v2
        if: always()
        with:
          files: pytest.xml
          check_name: Test Results
          check_run: false
          action_fail: true
          comment_mode: off

  # cibuildwheel build identifiers (cp310, ...) to build wheels for, sourced
  # from compatibility.yaml so the wheel matrix tracks the supported-python set
  # automatically. Full set on a release; just the min+max python on PRs/pushes
  # (a packaging smoke -- the full set is for the published artifacts). Emitted
  # as a JSON list so `wheels` fans out one (os, python) cell per wheel.
  wheel-matrix:
    needs: lint
    runs-on: ubuntu-latest
    outputs:
      cibw: ${{ steps.set.outputs.cibw }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          # dependencies: python.version_min
          python-version: "3.10"
      - run: pip install pyyaml
      - id: set
        env:
          MODE: ${{ github.event_name == 'release' && 'full' || 'pr' }}
        run: |
          echo "cibw=$(python scripts/compat_matrix.py expand --kind cibuildwheel --mode "$MODE")" >> "$GITHUB_OUTPUT"

  wheels:
    needs: [lint, wheel-matrix]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-26]
        python: ${{ fromJson(needs.wheel-matrix.outputs.cibw) }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v7
        with:
          submodules: recursive
      - name: Free disk space
        if: runner.os == 'Linux'
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          swap-storage: true
      - name: Install cibuildwheel
        run: pip install cibuildwheel
      - name: Build wheel
        # One python per matrix cell (matrix.python is a cibuildwheel build tag
        # like cp310) so the per-python builds run in PARALLEL instead of
        # serially within an OS job. Each cibuildwheel wheel reinstalls the
        # build-time torch inside the manylinux container (~24 min/wheel on
        # Linux), so the serial build was the dominant CI cost; wall time now
        # caps at one wheel regardless of python count.
        run: cibuildwheel --output-dir wheelhouse
        env:
          CIBW_BUILD: ${{ matrix.python }}-*
      - name: Resolve python version for the smoke test
        id: pyver
        shell: bash
        run: echo "version=3.${TAG#cp3}" >> "$GITHUB_OUTPUT"  # cp310 -> 3.10
        env:
          TAG: ${{ matrix.python }}
      - name: Set up Python for the wheel smoke test
        uses: actions/setup-python@v6
        with:
          python-version: ${{ steps.pyver.outputs.version }}
      - name: Smoke-test the built wheel
        # Install + import the wheel this cell just built. cibuildwheel's own
        # test-command only runs on tag releases, so this gives every
        # (os, python) cell a basic packaging check on PRs too. pip resolves the
        # runtime deps (torch / pyzag / nmhit) straight from pyproject.toml.
        shell: bash
        run: |
          set -euo pipefail
          python -m pip install --upgrade pip
          python -m pip install wheelhouse/*.whl
          python -c "import neml2; print(f'neml2 {neml2.__version__} loads from {neml2.__file__}')"
      - uses: actions/upload-artifact@v7
        if: github.event_name == 'release' && github.event.action == 'published'
        with:
          name: packages-wheels-${{ matrix.os }}-${{ matrix.python }}
          path: wheelhouse/*.whl

  sdist:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - name: Build SDist
        run: pipx run build --sdist
      - name: Upload SDist
        uses: actions/upload-artifact@v7
        if: github.event_name == 'release' && github.event.action == 'published'
        with:
          name: package-sdist
          path: dist/*.tar.gz

  PyPI:
    needs: [sdist, wheels]
    environment: pypi
    permissions:
      id-token: write
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'published'
    steps:
      - uses: actions/download-artifact@v8
        with:
          pattern: package-sdist
          path: dist
          merge-multiple: true
      - uses: actions/download-artifact@v8
        with:
          pattern: packages-wheels-*
          path: dist
          merge-multiple: true
      - uses: pypa/gh-action-pypi-publish@release/v1