Updates github actions timeout for longer tests #99
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main, advanced_preproc ] | |
| pull_request: | |
| branches: [ main ] | |
| workflow_call: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| code-quality: | |
| name: Code Quality Checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'pyproject.toml' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Check code formatting with Black | |
| run: | | |
| black --check --diff marEx/ | |
| - name: Check import sorting with isort | |
| run: | | |
| isort --check-only --diff marEx/ | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 marEx/ | |
| - name: Security check with bandit | |
| run: | | |
| bandit -r marEx/ -f json -o bandit-report.json || true | |
| - name: Type checking with mypy | |
| run: | | |
| mypy marEx/ || true | |
| - name: Upload bandit report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bandit-report | |
| path: bandit-report.json | |
| retention-days: 7 | |
| test: | |
| name: Test Suite | |
| runs-on: ${{ matrix.os }} | |
| needs: code-quality | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.10', '3.11', '3.12'] | |
| exclude: | |
| # Reduce matrix size for faster CI | |
| - os: windows-latest | |
| python-version: '3.10' | |
| - os: macos-latest | |
| python-version: '3.10' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| cache-dependency-path: 'pyproject.toml' | |
| - name: Install system dependencies (Ubuntu) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libhdf5-dev libnetcdf-dev libproj-dev proj-data proj-bin libgeos-dev | |
| - name: Install system dependencies (macOS) | |
| if: matrix.os == 'macos-latest' | |
| run: | | |
| brew install hdf5 netcdf proj geos | |
| - name: Install system dependencies (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| # Windows dependencies are handled by conda-forge packages | |
| echo "Windows dependencies handled by pip packages" | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev,full]" | |
| - name: Configure Dask for CI | |
| run: | | |
| mkdir -p ~/.dask | |
| echo "distributed:" > ~/.dask/config.yaml | |
| echo " worker:" >> ~/.dask/config.yaml | |
| echo " memory:" >> ~/.dask/config.yaml | |
| echo " target: 0.8" >> ~/.dask/config.yaml | |
| echo " spill: 0.9" >> ~/.dask/config.yaml | |
| echo " pause: 0.95" >> ~/.dask/config.yaml | |
| echo " terminate: 0.98" >> ~/.dask/config.yaml | |
| echo " admin:" >> ~/.dask/config.yaml | |
| echo " log-format: '%(name)s - %(levelname)s - %(message)s'" >> ~/.dask/config.yaml | |
| echo "array:" >> ~/.dask/config.yaml | |
| echo " chunk-size: 64MiB" >> ~/.dask/config.yaml | |
| - name: List installed packages | |
| run: pip list | |
| - name: Run unit tests | |
| run: | | |
| pytest tests/test_detect_helpers.py tests/test_track_helpers.py -n 2 -v --tb=short | |
| - name: Run error handling tests | |
| run: | | |
| pytest tests/test_error_handling.py -n 2 -v --tb=short | |
| - name: Run exception tests | |
| run: | | |
| pytest tests/test_exceptions.py -n 2 -v --tb=short | |
| - name: Run logging system tests | |
| run: | | |
| pytest tests/test_logging_system.py -n 2 -v --tb=short | |
| - name: Run preprocessing tests (gridded) | |
| run: | | |
| pytest tests/test_gridded_preprocessing.py -n 2 -v --tb=short | |
| - name: Run preprocessing tests (unstructured) | |
| run: | | |
| pytest tests/test_unstructured_preprocessing.py -n 2 -v --tb=short | |
| - name: Run tracking tests (gridded) | |
| run: | | |
| pytest tests/test_gridded_tracking.py -n 2 -v --tb=short | |
| - name: Run tracking tests (unstructured) | |
| run: | | |
| pytest tests/test_unstructured_tracking.py -n 2 -v --tb=short | |
| - name: Run plotting tests | |
| run: | | |
| pytest tests/test_plotx.py -n 2 -v --tb=short | |
| - name: Run integration tests | |
| run: | | |
| pytest tests/test_integration.py -n 2 -v --tb=short -m "not slow" | |
| coverage: | |
| name: Coverage Report | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: always() | |
| timeout-minutes: 150 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'pyproject.toml' | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libhdf5-dev libnetcdf-dev libproj-dev proj-data proj-bin libgeos-dev | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev,full]" | |
| - name: Configure Dask for Coverage (Conservative) | |
| run: | | |
| mkdir -p ~/.dask | |
| echo "distributed:" > ~/.dask/config.yaml | |
| echo " worker:" >> ~/.dask/config.yaml | |
| echo " memory:" >> ~/.dask/config.yaml | |
| echo " target: 0.5" >> ~/.dask/config.yaml | |
| echo " spill: 0.6" >> ~/.dask/config.yaml | |
| echo " pause: 0.7" >> ~/.dask/config.yaml | |
| echo " terminate: 0.8" >> ~/.dask/config.yaml | |
| echo " recent-to-old-time: 15s" >> ~/.dask/config.yaml | |
| echo " daemon: false" >> ~/.dask/config.yaml | |
| echo " scheduler:" >> ~/.dask/config.yaml | |
| echo " allowed-failures: 100" >> ~/.dask/config.yaml | |
| echo " work-stealing: false" >> ~/.dask/config.yaml | |
| echo " worker-ttl: 600s" >> ~/.dask/config.yaml | |
| echo " comm:" >> ~/.dask/config.yaml | |
| echo " timeouts:" >> ~/.dask/config.yaml | |
| echo " connect: 600s" >> ~/.dask/config.yaml | |
| echo " tcp: 600s" >> ~/.dask/config.yaml | |
| echo " retry:" >> ~/.dask/config.yaml | |
| echo " count: 20" >> ~/.dask/config.yaml | |
| echo " delay:" >> ~/.dask/config.yaml | |
| echo " min: 3s" >> ~/.dask/config.yaml | |
| echo " max: 60s" >> ~/.dask/config.yaml | |
| echo " admin:" >> ~/.dask/config.yaml | |
| echo " log-format: '%(name)s - %(levelname)s - %(message)s'" >> ~/.dask/config.yaml | |
| echo "array:" >> ~/.dask/config.yaml | |
| echo " chunk-size: 24MiB" >> ~/.dask/config.yaml | |
| echo " slicing:" >> ~/.dask/config.yaml | |
| echo " split-large-chunks: false" >> ~/.dask/config.yaml | |
| - name: Set environment for conservative execution | |
| run: | | |
| echo "DASK_NUM_WORKERS=1" >> $GITHUB_ENV | |
| echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV | |
| echo "MKL_NUM_THREADS=1" >> $GITHUB_ENV | |
| echo "NUMBA_NUM_THREADS=1" >> $GITHUB_ENV | |
| echo "PYTHONHASHSEED=42" >> $GITHUB_ENV | |
| echo "PYTEST_COVERAGE=true" >> $GITHUB_ENV | |
| - name: Generate coverage report | |
| timeout-minutes: 120 | |
| run: | | |
| # Run tests in parallel with coverage collection | |
| pytest tests/ -n auto --cov=marEx --cov-report=term --cov-report=html --cov-report=xml --cov-config=.coveragerc --tb=short -m "not nocov" -x --maxfail=3 | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| fail_ci_if_error: false | |
| verbose: true | |
| - name: Upload coverage HTML report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: htmlcov/ | |
| retention-days: 90 | |
| build-test: | |
| name: Build and Test Package | |
| runs-on: ubuntu-latest | |
| needs: test | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'pyproject.toml' | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install build twine | |
| - name: Build package | |
| run: | | |
| python -m build | |
| - name: Check package integrity | |
| run: | | |
| twine check dist/* | |
| - name: Test package installation | |
| shell: bash | |
| run: | | |
| pip install dist/*.whl | |
| python -c "import marEx; print(f'marEx version: {marEx.__version__}')" | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist-packages | |
| path: dist/ | |
| retention-days: 7 | |
| all-checks-passed: | |
| name: All Checks Passed | |
| runs-on: ubuntu-latest | |
| needs: [code-quality, test, coverage, build-test] | |
| if: always() | |
| steps: | |
| - name: Check all jobs status | |
| run: | | |
| if [[ "${{ needs.code-quality.result }}" == "success" && | |
| "${{ needs.test.result }}" == "success" && | |
| "${{ needs.coverage.result }}" == "success" && | |
| "${{ needs.build-test.result }}" == "success" ]]; then | |
| echo "All checks passed!" | |
| else | |
| echo "Some checks failed:" | |
| echo "Code Quality: ${{ needs.code-quality.result }}" | |
| echo "Tests: ${{ needs.test.result }}" | |
| echo "Coverage: ${{ needs.coverage.result }}" | |
| echo "Build Test: ${{ needs.build-test.result }}" | |
| exit 1 | |
| fi |