Update Lightning packaging to follow the PEP 517/518/621/660 standards (

#832) **Context:** This PR updates the Lightning Python packaging to the latest standard. This allows us to eliminate pip~=22.0 from requirements*.txt, comply with up-to-date build practices, and fix the inability to build Lightning in place on newer Python environments. **Description of the Change:** Most of the building configuration was moved to the ``pyproject.toml`` file. We kept the ``setup.py`` file working with setuptools in part of the configuration pre-processing. A script with some pre-pre-processing was added to configure the ``pyproject.toml`` file for each backend. **Benefits:** Follow the new standard. Allows the use of more up-to-date pip versions. There will be no impact on the final user, as the installation of the wheels is unaffected by the change in packaging. **Possible Drawbacks:** When building and installing Lightning backends from scratch, we need to run the configuration script. If not the installation will default to ``lightning.qubit``, or the last configuration. **Related GitHub Issues:** [sc-65227] --------- Co-authored-by: ringo-but-quantum <[email protected]> Co-authored-by: Lee James O'Riordan <[email protected]> Co-authored-by: Vincent Michaud-Rioux <[email protected]>
PennyLaneAI · Aug 12, 2024 · de1dc09 · de1dc09
1 parent d6f681d
commit de1dc09
Show file tree

Hide file tree

Showing 36 changed files with 402 additions and 275 deletions.
diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md
@@ -3,7 +3,7 @@
 ### New features since last release
 
 * Add `var` support to `lightning.tensor`. Note that `var` support is added via `obs**2` and this implementation scales `O(num_obs**2)`.
-  [(#804)](https://github.com/PennyLaneAI/pennylane-lightning/pull/804) 
+  [(#804)](https://github.com/PennyLaneAI/pennylane-lightning/pull/804)
 
 ### Breaking changes
 
@@ -16,6 +16,9 @@
 * Do not run GPU tests and Docker workflows on release.
   [(#788)](https://github.com/PennyLaneAI/pennylane-lightning/pull/788)
 
+* Update python packaging to follow PEP 517/518/621/660 standards.
+  [(#832)](https://github.com/PennyLaneAI/pennylane-lightning/pull/832)
+
 ### Improvements
 
 * `ENABLE_LAPACK` is off by default for all Lightning backends.

diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -38,7 +38,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt -y install python3 python3-pip && python -m pip install pip~=22.0 && python3 -m pip install -r requirements-dev.txt
+        run: sudo apt update && sudo apt -y install python3 python3-pip python-is-python3 && python -m pip install -r requirements-dev.txt
 
       - name: Run formatter
         run: PATH=$PATH:/home/ubuntu/.local/bin/:$(dirname $(which python3))/ ./bin/format --check ./pennylane_lightning/core/src
diff --git a/.github/workflows/tests_lgpu_python.yml b/.github/workflows/tests_lgpu_python.yml
@@ -64,7 +64,6 @@ jobs:
     strategy:
       matrix:
         pl_backend: ["lightning_gpu"]
-        default_backend: ["lightning_qubit"]
         cuda_version: ["12"]
 
     name: Python Tests (${{ matrix.pl_backend }}, cuda-${{ matrix.cuda_version }})
@@ -129,8 +128,7 @@ jobs:
           echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
           echo "Python_ROOT_DIR=${{ env.VENV_NAME }}" >> $GITHUB_ENV
           echo "Python3_ROOT_DIR=${{ env.VENV_NAME }}" >> $GITHUB_ENV
-          
-                    
+
           # Adding venv site-packages to output for subsequent step to referecen if needed
           echo "site_packages_dir=$(${{ env.VENV_NAME }}/bin/python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" >> $GITHUB_OUTPUT
 
@@ -179,12 +177,13 @@ jobs:
         run: |
           cd main
           rm -rf build
-          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.default_backend }} -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          PL_BACKEND=lightning_qubit python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
           python -m pip install . -vv
-          rm -rf build
 
           rm -rf build
-          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.pl_backend }} -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION) -DPython_EXECUTABLE=${{ steps.python_path.outputs.python }}" \
+          PL_BACKEND=${{ matrix.pl_backend }} python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION) -DPython_EXECUTABLE=${{ steps.python_path.outputs.python }}" \
           python -m pip install . -vv
 
       - name: Run PennyLane-Lightning-GPU unit tests

diff --git a/.github/workflows/tests_lgpumpi_python.yml b/.github/workflows/tests_lgpumpi_python.yml
@@ -110,7 +110,8 @@ jobs:
           python -m pip install -r requirements-dev.txt
 
           python -m pip install custatevec-cu${{ matrix.cuda_version_maj }} mpi4py openfermionpyscf
-          PL_BACKEND=lightning_qubit python -m pip install . -vv
+          PL_BACKEND=lightning_qubit python scripts/configure_pyproject_toml.py
+          python -m pip install . -vv
 
       - name: Checkout PennyLane for release build
         if: inputs.pennylane-version == 'release'
@@ -132,8 +133,9 @@ jobs:
           CUQUANTUM_SDK: $(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum')")
         run: |
           source /etc/profile.d/modules.sh && module use /opt/modules/ && module load ${{ matrix.mpilib }}/cuda-${{ matrix.cuda_version_maj }}.${{ matrix.cuda_version_min }}
+          PL_BACKEND=lightning_gpu python scripts/configure_pyproject_toml.py
           CMAKE_ARGS="-DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DENABLE_MPI=ON -DCMAKE_CUDA_COMPILER=$(which nvcc) -DCMAKE_CUDA_ARCHITECTURES=${{ env.CI_CUDA_ARCH }} -DPython_EXECUTABLE=${{ steps.python_path.outputs.python }}" \
-          PL_BACKEND=lightning_gpu python -m pip install . --verbose
+          python -m pip install . -vv
 
       # There are issues running py-cov with MPI. A solution is to use coverage as reported
       # [here](https://github.com/pytest-dev/pytest-cov/issues/237#issuecomment-544824228)

diff --git a/.github/workflows/tests_lkcpu_python.yml b/.github/workflows/tests_lkcpu_python.yml
@@ -117,13 +117,14 @@ jobs:
           cd main
           python -m pip install -r requirements-dev.txt
 
-      - name: Create device wheel
+      - name: Configure and build device wheel
         env:
           PL_BACKEND: ${{ matrix.pl_backend }}
         run: |
           cd main
-          python setup.py build_ext -i --define="PL_BACKEND=${{ matrix.pl_backend }};CMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos;ENABLE_PYTHON=ON;CMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)"
-          python setup.py bdist_wheel
+          python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          python -m build
           cd dist
           WHEEL_NAME=$(ls *.whl)
           cp $WHEEL_NAME ${{ github.workspace }}/wheel_${{ matrix.pl_backend }}-${{ matrix.exec_model }}.whl
@@ -211,6 +212,7 @@ jobs:
           python -m pip install -r requirements-dev.txt
           python -m pip install openfermionpyscf
           if [ '${{ inputs.lightning-version }}' != 'stable' ]; then
+            python scripts/configure_pyproject_toml.py
             SKIP_COMPILATION=True python -m pip install . -vv
           fi
           python -m pip install ${{ github.workspace }}/$WHEEL_NAME --no-deps

diff --git a/.github/workflows/tests_lkcuda_python.yml b/.github/workflows/tests_lkcuda_python.yml
@@ -241,8 +241,12 @@ jobs:
         run: |
           ls -l ${{ github.workspace }}/Kokkos
           cd main
-          SKIP_COMPILATION=True PL_BACKEND="lightning_qubit" python -m pip install . -vv
-          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.pl_backend }} -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+
+          PL_BACKEND=lightning_qubit python scripts/configure_pyproject_toml.py
+          SKIP_COMPILATION=True python -m pip install . -vv
+
+          PL_BACKEND=${{ matrix.pl_backend }} python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
           python -m pip install . -vv
 
       - name: Run PennyLane-Lightning unit tests
@@ -262,10 +266,13 @@ jobs:
         if: ${{ matrix.pl_backend == 'all' }}
         run: |
           cd main
-          CMAKE_ARGS="-DPL_BACKEND="lightning_qubit" -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+
+          PL_BACKEND=lightning_qubit python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
           python -m pip install . -vv
-          rm -rf build
-          CMAKE_ARGS="-DPL_BACKEND="lightning_kokkos" -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+
+          PL_BACKEND="lightning_kokkos" python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
           python -m pip install . -vv
 
       - name: Run PennyLane-Lightning unit tests for lightning.qubit with all devices installed

diff --git a/.github/workflows/tests_lmps_tncuda_python.yml b/.github/workflows/tests_lmps_tncuda_python.yml
@@ -60,20 +60,19 @@ jobs:
           echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
           nvcc --version
           nvidia-smi
-  
+
   pythontestswithMPSTNCuda:
     if: ${{ !contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
     needs: [builddeps]
     strategy:
       matrix:
         os: [ubuntu-22.04]
         pl_backend: ["lightning_tensor"]
-        default_backend: ["lightning_qubit"]
         pl_tensor_method: ["mps"]
         pl_tensor_backend: ["cutensornet"]
         cuda_version: ["12"]
 
-    name: Python Tests (${{ matrix.pl_backend }}, method-${{ matrix.pl_tensor_method }}, backend-${{ matrix.pl_tensor_backend }}, cuda-${{ matrix.cuda_version }})  
+    name: Python Tests (${{ matrix.pl_backend }}, method-${{ matrix.pl_tensor_method }}, backend-${{ matrix.pl_tensor_backend }}, cuda-${{ matrix.cuda_version }})
     runs-on:
       - ${{ matrix.os }}
       - self-hosted
@@ -92,14 +91,12 @@ jobs:
 
       - name: Checkout PennyLane-Lightning-Tensor-MPS-TNCuda
         uses: actions/checkout@v4
-        with:
-          path: main
 
       - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: '3.9'
-      
+
       # Since the self-hosted runner can be re-used. It is best to set up all package
       # installations in a virtual environment that gets cleaned at the end of each workflow run
       - name: Setup Python virtual environment
@@ -116,13 +113,13 @@ jobs:
 
           # Add the venv to PATH for subsequent steps
           echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
-          
+
           # Adding venv name as an output for subsequent steps to reference if needed
           source ${{ env.VENV_NAME }}/bin/activate
           echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
           echo "Python_ROOT_DIR=${{ env.VENV_NAME }}" >> $GITHUB_ENV
           echo "Python3_ROOT_DIR=${{ env.VENV_NAME }}" >> $GITHUB_ENV
-          
+
           # Adding venv site-packages to output for subsequent step to referecen if needed
           echo "site_packages_dir=$(${{ env.VENV_NAME }}/bin/python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" >> $GITHUB_OUTPUT
 
@@ -139,7 +136,6 @@ jobs:
 
       - name: Install required packages
         run: |
-          cd main
           python -m pip install -r requirements-dev.txt
           python -m pip install ninja cmake scipy custatevec-cu${{ matrix.cuda_version }} cutensornet-cu${{ matrix.cuda_version }} openfermionpyscf
 
@@ -157,29 +153,25 @@ jobs:
           git fetch --all
           git checkout $(git branch -a --list "origin/v0.*rc*" | sort | tail -1)
           python -m pip uninstall -y pennylane && python -m pip install . -vv --no-deps
-  
+
       - name: Install Stable PennyLane
         if: inputs.pennylane-version == 'stable'
         run: |
-          cd main
           python -m pip uninstall -y pennylane && python -m pip install -U pennylane
-  
-      - name: Build and install package
+
+      - name: Configure and install required backends
         run: |
-          cd main
-          rm -rf build
-          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.default_backend }} -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
-          python -m pip install . -vv
           rm -rf build
-  
+          PL_BACKEND=lightning_qubit python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" python -m pip install . -vv
+
           rm -rf build
-          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.pl_backend }} -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
-          python -m pip install . -vv
-  
+          PL_BACKEND="${{ matrix.pl_backend }}" python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" python -m pip install . -vv
+
       - name: Run PennyLane-Lightning-Tensor unit tests
         if: ${{ matrix.pl_backend != 'all'}}
         run: |
-          cd main/
           DEVICENAME=`echo ${{ matrix.pl_backend }} | sed "s/_/./g"`
           PL_DEVICE=${DEVICENAME} python -m pytest tests $COVERAGE_FLAGS
           mv coverage.xml coverage-${{ github.job }}-${{ matrix.pl_backend }}.xml
@@ -188,7 +180,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ubuntu-codecov-results-python
-          path: ./main/coverage-${{ github.job }}-${{ matrix.pl_backend }}.xml
+          path: coverage-${{ github.job }}-${{ matrix.pl_backend }}.xml
           if-no-files-found: error
 
 

diff --git a/.github/workflows/tests_lqcpu_python.yml b/.github/workflows/tests_lqcpu_python.yml
@@ -86,8 +86,9 @@ jobs:
         id: create_lq_wheel
         run: |
           cd main
-          python setup.py build_ext -i --define="PL_BACKEND=${{ matrix.pl_backend }};ENABLE_BLAS=${{ matrix.blas }};LQ_ENABLE_KERNEL_OMP=ON;ENABLE_PYTHON=ON;CMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)"
-          PL_BACKEND=${{ matrix.pl_backend }} python setup.py bdist_wheel
+          PL_BACKEND=${{ matrix.pl_backend }} python scripts/configure_pyproject_toml.py
+          CMAKE_ARGS="-DENABLE_BLAS=${{ matrix.blas }} -DLQ_ENABLE_KERNEL_OMP=ON -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          python -m build
           cd dist
           WHEEL_NAME=$(ls *.whl)
           cp $WHEEL_NAME ${{ github.workspace }}/wheel_${{ matrix.pl_backend }}-${{ matrix.blas }}.whl

diff --git a/.github/workflows/tests_windows_cpp.yml b/.github/workflows/tests_windows_cpp.yml
@@ -114,7 +114,7 @@ jobs:
           python-version: '3.9'
 
       - name: Checkout PennyLane-Lightning
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/tests_without_binary.yml b/.github/workflows/tests_without_binary.yml
@@ -46,19 +46,16 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-tags: true
-          path: main
 
       - name: Switch to release build of Lightning
         if: inputs.lightning-version == 'release'
         run: |
-          cd main
           git fetch --all
           git checkout $(git branch -a --list "origin/v0.*rc*" | sort | tail -1)
 
       - name: Switch to stable build of Lightning
         if: inputs.lightning-version == 'stable'
         run: |
-          cd main
           git fetch --tags --force
           git checkout latest_release
           git log -1 --format='%H'
@@ -72,7 +69,6 @@ jobs:
       - name: Get required Python packages
         run: |
           rm -fr $(python -m pip cache dir)/selfcheck/
-          cd main
           python -m pip install -r requirements-dev.txt
 
       - name: Checkout PennyLane for release build
@@ -93,39 +89,34 @@ jobs:
       - name: Install Stable PennyLane
         if: inputs.pennylane-version == 'stable'
         run: |
-          cd main
           python -m pip uninstall -y pennylane && python -m pip install -U pennylane
 
-      - name: Install the pennylane_lightning package
-        if: ${{ contains(fromJson('["lightning_kokkos", "lightning_gpu", "lightning_tensor"]'), matrix.pl_backend) }}
+      - name: Install the pennylane_lightning core package (lightning_qubit)
+        env:
+          SKIP_COMPILATION: True
+          PL_BACKEND: "lightning_qubit"
         run: |
-          cd main
-          SKIP_COMPILATION=True PL_BACKEND="lightning_qubit" python -m pip install . -vv
+          python scripts/configure_pyproject_toml.py
+          python -m pip install . -vv
 
-      - name: Install backend device
-        # FIXME: Remove this condition with v0.37.0 release
-        if: matrix.pl_backend != 'lightning_tensor' || inputs.lightning-version != 'stable'
+      - name: Install extra backend device
+        if: matrix.pl_backend != 'lightning_qubit'
         env:
           SKIP_COMPILATION: True
           PL_BACKEND: ${{ matrix.pl_backend }}
         run: |
-          cd main
+          python scripts/configure_pyproject_toml.py
           python -m pip install . -vv
 
       - name: Run PennyLane-Lightning unit tests for all backends
-        # FIXME: Remove this condition with v0.37.0 release
-        if: matrix.pl_backend != 'lightning_tensor' || inputs.lightning-version != 'stable'
         run: |
-          cd main/
           DEVICENAME=`echo ${{ matrix.pl_backend }} | sed "s/_/./g"`
           PL_DEVICE=${DEVICENAME} python -m pytest tests/ $COVERAGE_FLAGS
 
       - name: Upload coverage to Codecov
-        # FIXME: Remove this condition with v0.37.0 release
-        if: matrix.pl_backend != 'lightning_tensor' || inputs.lightning-version != 'stable'
         uses: codecov/codecov-action@v4
         with:
-          files: ./main/coverage.xml
+          files: coverage.xml
           fail_ci_if_error: true
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/wheel_linux_aarch64.yml b/.github/workflows/wheel_linux_aarch64.yml
@@ -119,8 +119,11 @@ jobs:
           mkdir Kokkos
           cp -rf ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}/* Kokkos/
 
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel~=2.16.0
+      - name: Install dependencies
+        run: python -m pip install cibuildwheel~=2.16.0 toml
+
+      - name: Configure pyproject.toml file
+        run: PL_BACKEND="${{ matrix.pl_backend }}" python scripts/configure_pyproject_toml.py
 
       - uses: docker/setup-qemu-action@v2
         name: Set up QEMU
@@ -138,15 +141,19 @@ jobs:
             cat /etc/yum.conf | sed "s/\[main\]/\[main\]\ntimeout=5/g" > /etc/yum.conf
             python -m pip install ninja cmake~=3.24.0
 
-          CIBW_ENVIRONMENT: PL_BACKEND="${{ matrix.pl_backend }}" CMAKE_ARGS="-DENABLE_LAPACK=OFF"
+          CIBW_ENVIRONMENT: CMAKE_ARGS="-DENABLE_LAPACK=OFF"
 
           CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
 
           CIBW_BUILD_VERBOSITY: 3
 
           CIBW_BEFORE_TEST: |
             python -m pip install -r requirements-tests.txt
-            if ${{ matrix.pl_backend == 'lightning_kokkos'}}; then SKIP_COMPILATION=True PL_BACKEND="lightning_qubit" python -m pip install . -vv; fi
+            if ${{ matrix.pl_backend == 'lightning_kokkos'}}
+            then
+              PL_BACKEND="lightning_qubit" python scripts/configure_pyproject_toml.py
+              SKIP_COMPILATION=True python -m pip install . -vv
+            fi
 
           CIBW_TEST_COMMAND: |
             DEVICENAME=`echo ${{ matrix.pl_backend }} | sed "s/_/./g"`