ci: Modernize CI infra

* Split workflows into logical steps and use nested workflow approach

* General maintainence on workflows to work with new build organization and build system

* Install kernel when quatro is installed. Seems like CI fails otherwise
https://github.com/mahendrapaipuri/jupytext/actions/runs/6561210848/job/17820549979

* Add alls-green action to show CI results in more UI friendly way

* Use optional uploading of build artifacts

.ci/environment-ci.yml

@@ -1,11 +1,15 @@
 # This environment is smaller than the dev environment.
-# In particular it does not include JupyterLab nor the community notebook extensions
+#
+# Until conda supports creating env from pyproject.toml
+# (https://github.com/conda/conda/pull/12666)
+# we need to maintain environment-ci.yml file that is inline with deps in
+# pyproject.toml
+#
 name: jupytext-ci
 channels:
   - defaults
   - conda-forge
 dependencies:
-  - jupyter
   - ipykernel
   - nbconvert
   - nbformat>=5.1.2

.github/workflows/ci.yml

@@ -0,0 +1,62 @@
+name: CI
+on:
+  workflow_dispatch:
+    inputs:
+      upload-build-artifacts:
+        type: boolean
+        required: false
+        default: false
+        description: Upload build artifacts
+  push:
+    paths-ignore:
+      - 'CHANGELOG.md'
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 11 * * 4'
+
+permissions:
+  # All nested workflows will inherit these permissions and so no need to declare
+  # in each step file
+  contents: read
+  # Cannot use it in codeql nested workflow without declaring it on
+  # top level workflow
+  # Ref: https://docs.github.com/en/actions/using-workflows/reusing-workflows#access-and-permissions
+  security-events: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pre-commit:
+    uses: ./.github/workflows/step_pre-commit.yml
+
+  codeql:
+    needs: [ pre-commit ]
+    uses: ./.github/workflows/step_static-analysis.yml
+
+  test-pip:
+    needs: [ codeql ]
+    uses: ./.github/workflows/step_tests-pip.yml
+
+  test-conda:
+    needs: [ codeql ]
+    uses: ./.github/workflows/step_tests-conda.yml
+
+  build:
+    needs: [ test-pip, test-conda ]
+    uses: ./.github/workflows/step_build.yml
+    with:
+      upload: ${{ inputs.upload-build-artifacts || false }}
+
+  pass:
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check jobs
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+    if: always()

.github/workflows/publish.yml

@@ -7,23 +7,47 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+-dev[0-9]+"
 
 jobs:
+  pre-commit:
+    uses: ./.github/workflows/step_pre-commit.yml
+
+  codeql:
+    needs: [ pre-commit ]
+    uses: ./.github/workflows/step_static-analysis.yml
+
+  test-pip:
+    needs: [ codeql ]
+    uses: ./.github/workflows/step_tests-pip.yml
+
+  test-conda:
+    needs: [ codeql ]
+    uses: ./.github/workflows/step_tests-conda.yml
+
+  build:
+    needs: [ test-pip, test-conda ]
+    uses: ./.github/workflows/step_build.yml
+
   publish:
+    needs: [ build ]
     runs-on: ubuntu-latest
+
     environment:
       name: pypi
       url: https://pypi.org/p/jupytext
+
     permissions:
       id-token: write
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v3
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
       - name: Build package
         run: |
-          python -m pip install wheel jupyter-packaging 'jupyterlab>=3,<4'
-          BUILD_JUPYTERLAB_EXTENSION=1 python setup.py sdist bdist_wheel
+          python -m pip install wheel build
+          python -m build
+
       - name: Publish
         uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/step_build.yml

@@ -0,0 +1,58 @@
+name: build
+run-name: Build test
+
+on:
+  workflow_call:
+    inputs:
+      upload:
+        type: boolean
+        required: false
+        default: false
+        description: Upload build artifacts
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-build
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Build package
+        run: |
+          python -m pip install build wheel
+
+          # NOTE: These builds and verifications of the builds can be done more
+          # robustily with jupyter-releaser.
+          #
+          # Removed the check on size of package as we are distributing tests/ with
+          # sdist now and they are around 8MB. Seems like original check was to make
+          # sure we are not distributing node_modules and we are quite safe with that
+          # with hatch build system.
+          #
+          # Build jupytext package
+          python -m build
+
+          # Build lab extension(s)
+          npm pack --pack-destination dist jupyterlab/packages/*
+
+          # Check that the lab is there
+          if (($(tar -tf dist/*.tar.gz | grep jupyterlab/jupyterlab_jupytext/labextension/package.json$ | wc -l)==0)); then echo "Missing jupyterlab_jupytext" && exit 1; fi
+
+          # Install package and extension
+          python -m pip install dist/*.tar.gz
+
+          echo "Install went OK"
+
+      - name: Archive build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+        if: ${{ inputs.upload }}