Simplify dev tooling, remove colorcet examples, drop 3.7, add 3.12 (#…

…120)

* use mostly pip on the CI
* remove colorcet examples cmd
* move to pyproject
* improve git archive
* remove unused manifest
* simplify building the docs
* update dev instructions
* drop 3.7 and try all python versions
* simplify version definition

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+include = *.py
+exclude = .git,__pycache__,.tox,.eggs,*.egg,doc,dist,build,_build,.ipynb_checkpoints,run_test.py
+ignore = E,
+         W

.git_archival.txt

@@ -0,0 +1,4 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
+ref-names: $Format:%D$

.gitattributes

@@ -1,2 +1,5 @@
-colorcet/__init__.py export-subst
-setup.cfg export-subst
+# For setuptools_scm with .git_archival.txt
+.git_archival.txt  export-subst
+# Line Endings configuration file for Git
+# Set the default behavior, in case people don't have or can't have core.autocrlf set.
+* text=auto

.github/workflows/build.yaml

@@ -19,92 +19,57 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
-    env:
-      CHANS: "-c pyviz -c bokeh"
-      PKG_TEST_PYTHON: "--test-python=py37"
-      PYTHON_VERSION: "3.7"
-      MPLBACKEND: "Agg"
-      CONDA_UPLOAD_TOKEN: ${{ secrets.CONDA_UPLOAD_TOKEN }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: "100"
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v4
+      - name: Fetch unshallow
+        run: git fetch --prune --tags --unshallow -f
       - uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Fetch
-        run: git fetch --prune --tags -f
+          python-version: "3.11"
+          auto-update-conda: true
       - name: Set output
         id: vars
         run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
       - name: conda setup
         run: |
-          conda config --set always_yes True
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
+          conda install anaconda-client conda-build setuptools_scm
       - name: conda build
-        run: doit package_build $CHANS $PKG_TEST_PYTHON --test-group=unit
+        run: |
+          VERSION=`python -m setuptools_scm` conda build conda.recipe/
       - name: conda dev upload
         if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev
+        run: |
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe)
       - name: conda main upload
         if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --label=main
+        run: |
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe)
   pip_build:
     name: Build PyPI Packages
     runs-on: 'ubuntu-latest'
     defaults:
       run:
         shell: bash -l {0}
-    env:
-      CHANS: "-c pyviz -c bokeh"
-      PKG_TEST_PYTHON: "--test-python=py37"
-      PYTHON_VERSION: "3.7"
-      MPLBACKEND: "Agg"
-      PPU: ${{ secrets.PPU }}
-      PPP: ${{ secrets.PPP }}
-      PYPI: "https://upload.pypi.org/legacy/"
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: "100"
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/checkout@v4
+      - name: Fetch unshallow
+        run: git fetch --prune --tags --unshallow -f
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          miniconda-version: "latest"
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Fetch
-        run: git fetch --prune --tags -f
-      - name: conda setup
-        run: |
-          conda config --set always_yes True
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
-          doit env_create $CHANS --python=$PYTHON_VERSION
+          python-version: "3.11"
       - name: env setup
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit develop_install $CHANS_DEV -o unit_tests
-          doit pip_on_conda
-      - name: doit env_capture
-        run: |
-          conda activate test-environment
-          doit env_capture
+          python -m pip install --upgrade pip
+          python -m pip install build
       - name: pip build
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_build
-      - name: pip upload
+          python -m build
+      - name: Publish package to PyPI
         if: github.event_name == 'push'
-        run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_upload -u $PPU -p $PPP -r $PYPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: ${{ secrets.PPU }}
+          password: ${{ secrets.PPP }}
+          packages-dir: dist/

.github/workflows/docs.yaml

@@ -31,42 +31,25 @@ jobs:
       run:
         shell: bash -l {0}
     env:
-      DESC: "Documentation build"
       MPLBACKEND: "Agg"
       DISPLAY: ":99.0"
-      PYTHON_DOCS_VERSION: "3.9"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "100"
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: "latest"
       - name: Fetch
         run: git fetch --prune --tags -f
       - name: Set output
         id: vars
         run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
-      - name: conda setup
-        # conda-forge is required to install pydata-sphinx-theme and sphinx-copybutton.
-        run: |
-          conda create -n test-environment
-          conda activate test-environment
-          conda config --env --append channels pyviz/label/dev --append channels conda-forge
-          conda config --env --remove channels defaults
-          conda install python=${{ env.PYTHON_DOCS_VERSION}} pyctdev
-      - name: doit develop_install
-        run: |
-          conda activate test-environment
-          doit develop_install -o doc -o examples
-      - name: doit env_capture
-        run: |
-          conda activate test-environment
-          doit env_capture
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: install
+        run: pip install ."[doc, examples]"
       - name: build docs
-        run: |
-          conda activate test-environment
-          doit build_website
+        run: sphinx-build -b html doc builtdocs
       - name: git status
         run: |
           git status

.github/workflows/test.yaml

@@ -16,71 +16,51 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  pre_commit:
+    name: Run pre-commit
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/[email protected]
   test_suite:
     name: Pytest on ${{ matrix.python-version }}, ${{ matrix.os }}
+    needs: [pre_commit]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
         # Run on the full set on schedule, workflow_dispatch and push&tags events, otherwise on a subset.
-        python-version: ${{ ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || ( github.event_name == 'push' && github.ref_type == 'tag' ) ) && fromJSON('["3.7", "3.8", "3.9", "3.10", "3.11"]') || fromJSON('["3.7", "3.11"]') }}
+        # python-version: ${{ ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || ( github.event_name == 'push' && github.ref_type == 'tag' ) ) && fromJSON('["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]') || fromJSON('["3.7", "3.12"]') }}
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
     timeout-minutes: 60
     defaults:
       run:
-        shell: bash -l {0} 
+        shell: bash -l {0}
     env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      ENV_NAME: "colorcet"
       MPLBACKEND: "Agg"
       DISPLAY: ":99.0"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "100"
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: "latest"
       - name: Fetch
         run: git fetch --prune --tags -f
-      - name: conda setup
-        run: |
-          conda update -n base conda
-          conda create -n ${{ env.ENV_NAME }}
-          conda activate ${{ env.ENV_NAME }}
-          conda config --env --prepend channels pyviz/label/dev
-          conda config --env --show-sources
-          conda install python=${{ matrix.python-version }} pyctdev
-      - name: doit develop_install
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          doit develop_install -o tests -o examples
-      - name: pin bokeh on Python 3.8
-        if: matrix.python-version == '3.8'
-        # Pin to avoid pulling an old version of Bokeh and Panel.
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          conda install "bokeh >=3"
-      - name: doit env_capture
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          doit env_capture
-      - name: doit test_lint
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          doit test_lint
-      - name: doit test_unit
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          doit test_unit
-      - name: doit test_examples
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          doit test_examples
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+      - name: pip update
+        run: pip install --upgrade pip
+      - name: install
+        run: pip install -e ."[tests,tests_examples,tests_extra]"
+      - name: pip list
+        run: pip list
+      - name: unit tests
+        run: pytest colorcet --cov=colorcet   --cov-append --cov-report xml
+      - name: examples tests
+        run: pytest doc --nbval-lax -p no:python
       - name: doit test_unit_extra
-        run: |
-          conda activate ${{ env.ENV_NAME }}
-          pip install pytest-mpl
-          doit test_unit_extra
-      - uses: codecov/codecov-action@v3
+        run: pytest colorcet --mpl
+      - uses: codecov/codecov-action@v4
         if: github.event_name == 'push'

.gitignore

@@ -18,21 +18,23 @@ coverage.xml
 __pycache__
 .doit*
 *.egg-info
-*/.version
 pip-wheel-metadata
 **.ipynb_checkpoints
 /build
+dist/
+.venv/
+venv/
 
 # nbsite
 # these files normally shouldn't be checked in as they should be
 # dynamically built from notebooks
-doc/**/*.rst
-doc/**/*.ipynb
-doc/**/*.json
 # this dir contains the whole website and should not be checked in on main
 builtdocs/
 # myst-nb output dir
 jupyter_execute/
 
 # CET_updates.py is an intermediate file and should not be versioned.
 assets/CET_updates.py
+
+# setuptools_scm
+colorcet/_version.py

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+# This is the configuration for pre-commit, a local framework for managing pre-commit hooks
+#   Check out the docs at: https://pre-commit.com/
+
+default_stages: [commit]
+
+# Exclude Matlab files
+exclude: '.+\.m$'
+
+repos:
+-   repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.7.1
+    hooks:
+    -   id: nbqa-flake8
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-builtin-literals
+    -   id: check-case-conflict
+    -   id: check-docstring-first
+    -   id: check-executables-have-shebangs
+    -   id: check-toml
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+        exclude: (\.min\.js$|\.svg$)
+    -   id: trailing-whitespace
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+    -   id: flake8  # See 'setup.cfg' for args
+        args: [colorcet]
+        files: colorcet/
+-   repo: https://github.com/hoxbro/clean_notebook
+    rev: v0.1.15a2
+    hooks:
+    - id: clean-notebook

README.md

@@ -28,7 +28,7 @@ by Peter Kovesi at the Center for Exploration Targeting.
 
 ## Installation
 
-Colorcet supports Python 3.7, 3.8, 3.9, 3.10 and 3.11  on Linux, Windows, or Mac
+Colorcet supports Python 3.7 and greater on Linux, Windows, or Mac
 and can be installed with conda:
 
 ```sh
@@ -41,16 +41,6 @@ or with pip:
 python -m pip install colorcet
 ```
 
-Once installed you can copy the examples into the current directory using the colorcet command and run them using the Jupyter notebook:
-
-```sh
-colorcet examples
-cd colorcet-examples
-jupyter notebook
-```
-
-(Here colorcet examples is a shorthand for colorcet copy-examples --path colorcet-examples && colorcet fetch-data --path colorcet-examples.)
-
 To work with JupyterLab you will also need the PyViz JupyterLab extension:
 
 ```sh