tests.yaml

name: Tests

on:
  pull_request:
  workflow_dispatch:
    inputs:
      suite:
        description: Test suite to run
        required: true
        default: good-enough
        type: choice
        options:
          - good-enough
          - exhaustive
          - coverage

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# Snapshot fixture paths exceed Git for Windows' legacy checkout limit.
env:
  GIT_CONFIG_COUNT: "1"
  GIT_CONFIG_KEY_0: core.longpaths
  GIT_CONFIG_VALUE_0: "true"

defaults:
  run:
    shell: bash

jobs:
  lightweight:
    name: Good-enough (${{ matrix.os }}, Python ${{ matrix.python-version }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: ${{ matrix.os == 'windows-latest' && 40 || 20 }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
        include:
          - os: windows-latest
            python-version: "3.12"
          - os: macos-latest
            python-version: "3.12"

    steps:
      - uses: actions/checkout@v5

      - name: Install Bash for named test suites
        if: runner.os == 'macOS'
        env:
          HOMEBREW_NO_AUTO_UPDATE: "1"
        run: |
          brew install bash
          echo "$(brew --prefix)/bin" >> "$GITHUB_PATH"

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install lightweight dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r test_requirements.txt ".[test]"

      - name: Enable real-kernel notebook coverage on Linux Python 3.12
        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
        run: python -m pip install "ipykernel==7.2.0" "jupyter-client==8.8.0"

      - name: Show resolved versions
        run: |
          python - <<'PY'
          import platform
          from importlib import metadata

          print(f"python: {platform.python_version()}")
          print(f"dryml: {metadata.version('dryml')}")
          PY

      - name: Run representative platform tests
        env:
          REQUESTED_SUITE: ${{ github.event_name == 'workflow_dispatch' && inputs.suite || 'good-enough' }}
        run: |
          if [[ "$RUNNER_OS" == "Linux" && "$REQUESTED_SUITE" != "good-enough" ]]; then
            bash ./tests.sh medium --ignore tests/old --ignore tests/dev -x -vv --timeout=180 --timeout-method=thread
          else
            bash ./tests.sh good-enough --ignore tests/old --ignore tests/dev -x -vv --timeout=180 --timeout-method=thread
          fi

  heavy:
    name: Heavy frameworks (Ubuntu, Python 3.12)
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      JAX_PLATFORMS: cpu
      PYTHONDONTWRITEBYTECODE: "1"
      OMP_NUM_THREADS: "1"
      OPENBLAS_NUM_THREADS: "1"
      MKL_NUM_THREADS: "1"
      BLIS_NUM_THREADS: "1"
      NUMEXPR_NUM_THREADS: "1"
      VECLIB_MAXIMUM_THREADS: "1"

    steps:
      - uses: actions/checkout@v5

      - name: Set up Python 3.12
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install all heavy frameworks with pip and build DRYML
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r test_requirements.txt -r heavy_test_requirements.txt \
            ".[test,tf,torch,jax,sklearn,xgboost,statsmodels,ray]"
          python -m build --no-isolation --outdir "$RUNNER_TEMP/heavy-artifact"
          python -m pip install --force-reinstall --no-deps "$RUNNER_TEMP"/heavy-artifact/*.whl
          python -m pip check

      - name: Require all CPU frameworks and exact native readers
        env:
          CUDA_VISIBLE_DEVICES: ""
        run: |
          python - <<'PY'
          from importlib import metadata
          import platform
          import jax
          from flax import nnx
          import optax
          import tensorflow
          import tensorflow_datasets
          import torch
          import ray
          import sklearn
          import xgboost
          from statsmodels.regression.linear_model import OLS
          from statsmodels.discrete.discrete_model import Logit

          assert jax.default_backend() == "cpu"
          assert jax.devices() and all(device.platform == "cpu" for device in jax.devices())
          assert isinstance(nnx.Linear(1, 1, rngs=nnx.Rngs(0)), nnx.Module)
          assert optax.adam(1e-3) is not None
          assert torch.version.cuda is None
          torch.optim.SGD([torch.nn.Parameter(torch.ones(1))], lr=0.1)
          assert OLS is not None and Logit is not None
          expected = {"numpy": "2.1.3", "scipy": "1.18.0", "pandas": "3.0.5",
                      "xgboost": "3.4.1", "scikit-learn": "1.9.0", "statsmodels": "0.15.0"}
          for package, required in expected.items():
              assert metadata.version(package) == required, (package, metadata.version(package))
          print(f"python: {platform.python_version()}; platform: {platform.platform()}")
          for package in ("dryml", "tensorflow", "tensorflow-datasets", "torch", "jax",
                          "jaxlib", "flax", "optax", "ray", *expected):
              print(f"{package}: {metadata.version(package)}")
          PY

      - name: Run framework-specific tests
        env:
          DRYML_TEST_BOOTSTRAP_CONTEXTS: "1"
          CUDA_VISIBLE_DEVICES: ""
        run: |
          bash ./tests.sh @tests/framework_tests.txt \
            --no-cov -vv --timeout=600 --timeout-method=thread \
            --basetemp="$RUNNER_TEMP/heavy-pytest" \
            -o cache_dir="$RUNNER_TEMP/heavy-cache" -o junit_family=legacy \
            --junitxml="$RUNNER_TEMP/heavy-evidence/results.xml" \
            --dryml-timing-output="$RUNNER_TEMP/heavy-evidence/executed.json"

      # Native tests need neither context bootstrap nor CUDA device overrides.
      - name: Run native-estimator tests in a fresh interpreter
        env:
          DRYML_TEST_INSTALLED_ESTIMATORS: "1"
        run: |
          bash ./tests.sh @tests/native_estimator_tests.txt \
            --no-cov -vv --timeout=600 --timeout-method=thread \
            --basetemp="$RUNNER_TEMP/native-pytest" \
            -o cache_dir="$RUNNER_TEMP/native-cache" -o junit_family=legacy \
            --junitxml="$RUNNER_TEMP/heavy-evidence/native-results.xml" \
            --dryml-timing-output="$RUNNER_TEMP/heavy-evidence/native-executed.json"

      - name: Require all 97 native-estimator cases without skips
        run: |
          python - <<'PY'
          import os
          from pathlib import Path
          import xml.etree.ElementTree as ET

          native = ET.parse(Path(os.environ['RUNNER_TEMP']) / 'heavy-evidence/native-results.xml').findall('.//testcase')
          assert len(native) == 97, len(native)
          assert not any(case.find(tag) is not None for case in native for tag in ('skipped', 'failure', 'error'))
          print('97 mixed-environment native-estimator cases passed, zero skips')
          PY

      - name: Run remaining heavy combinations only on explicit request
        if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'exhaustive' || inputs.suite == 'coverage')
        run: |
          excluded=()
          for selection in tests/framework_tests.txt tests/native_estimator_tests.txt; do
            while IFS= read -r path; do
              excluded+=(--deselect "$path")
            done < "$selection"
          done
          bash ./tests.sh heavy "${excluded[@]}" --ignore tests/old --ignore tests/dev \
            -x -vv --timeout=600 --timeout-method=thread

      - name: Retain framework execution and JUnit evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: heavy-ubuntu-py312
          path: ${{ runner.temp }}/heavy-evidence
          if-no-files-found: warn

  ray-integration:
    name: Existing Ray integration (Ubuntu, Python 3.12)
    if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'exhaustive' || inputs.suite == 'coverage')
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}

    steps:
      - uses: actions/checkout@v5

      - name: Set up coordinator Conda environment
        uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: dryml-ray
          auto-activate-base: false
          python-version: "3.12"

      - name: Build and install the test artifact in existing targets
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r ray_test_requirements.txt
          python -m build
          python -m pip install --force-reinstall dist/*.whl
          export VENV_ROOT="$RUNNER_TEMP/dryml-ray-venv"
          python -m venv "$VENV_ROOT"
          export VENV_PYTHON="$VENV_ROOT/bin/python"
          "$VENV_PYTHON" -m pip install --upgrade pip
          "$VENV_PYTHON" -m pip install -r ray_test_requirements.txt
          "$VENV_PYTHON" -m pip install --force-reinstall dist/*.whl

      - name: Start job-owned Ray test server
        run: |
          ray start --head --port=6379 --num-cpus=4 --num-gpus=0 --object-store-memory=100000000 --temp-dir="$RUNNER_TEMP/dryml-ray" --disable-usage-stats

      - name: Run enabled existing-Ray integration
        run: |
          export CONDA_PREFIX
          export VENV_PYTHON="$RUNNER_TEMP/dryml-ray-venv/bin/python"
          export DRYML_EXECUTE_INTEGRATION=1
          export DRYML_TEST_RAY_ADDRESS=127.0.0.1:6379
          export DRYML_TEST_CONDA_PREFIX="$CONDA_PREFIX"
          export DRYML_TEST_VENV_PYTHON="$VENV_PYTHON"
          ./tests.sh tests/ray/test_core_execute_ray.py tests/ray/test_execute_ray_backend.py tests/ray/test_execute_ray_environments.py tests/ray/test_dispatch_ray.py tests/core/test_execute_integration.py tests/execute/test_backend_conformance.py tests/execute/test_existing_environments.py --ignore tests/old --ignore tests/dev -x

      - name: Stop job-owned Ray test server
        if: always()
        run: ray stop --force

  coverage:
    name: Coverage (Ubuntu, Python 3.12)
    if: github.event_name == 'workflow_dispatch' && inputs.suite == 'coverage'
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v5

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install test and heavy dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r test_requirements.txt -r heavy_test_requirements.txt
          python -m pip install .

      - name: Run combined coverage
        run: |
          # Exact native readers are verified without instrumentation in heavy.
          ./tests.sh coverage --ignore tests/old --ignore tests/dev -x -vv --timeout=180 --timeout-method=thread \
            --deselect tests/models/test_xgb_models.py \
            --deselect tests/models/test_statsmodels_models.py \
            --deselect tests/models/test_estimator_experiments.py \
            --deselect tests/models/test_experiment_recovery.py::test_native_terminal_replay_reemits_exact_associated_checkpoint \
            --deselect tests/models/test_experiment_recovery.py::test_native_preassociation_failure_retains_live_fit_but_prior_authority_refits \
            --deselect tests/core/test_state_ref_restore_into.py::test_native_graph_restore_unfitted_fitted_invalidation_and_fresh_recovery