Update ALL ubuntu based workflows to use large runners (no label) (#967)

**Context:** Lightning uses a large amount of the runner pool, this PR ports lightning to using the large runner pool all the time (no label necessary). **Description of the Change:** `determine-workflow-runner` was removed, the large runner pool is verbatim used wherever necessary. **Benefits:** More storage space available to workflows (150GB vs 15GB of standard runners) **Possible Drawbacks:** Huge cost implications, but we will need to monitor and see how it goes! **Related GitHub Issues:** [sc-76978](https://app.shortcut.com/xanaduai/story/76978/move-lightning-over-to-using-large-runners-all-the-time) --------- Co-authored-by: ringo-but-quantum <[email protected]>
PennyLaneAI · Nov 4, 2024 · f909504 · f909504
1 parent 5f70b06
commit f909504
Show file tree

Hide file tree

Showing 11 changed files with 30 additions and 137 deletions.
diff --git a/.github/workflows/determine-workflow-runner.yml b/.github/workflows/determine-workflow-runner.yml
diff --git a/.github/workflows/tests_linux_cpp.yml b/.github/workflows/tests_linux_cpp.yml
@@ -39,16 +39,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-22.04
-
   cpptests:
     if: ${{ !contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
-    needs: [determine_runner]
     strategy:
       matrix:
         pl_backend: ["lightning_qubit"]
@@ -60,7 +52,7 @@ jobs:
           enable_kernel_avx_streaming: ON
     timeout-minutes: 60
     name: C++ Tests (${{ matrix.pl_backend }}, ENABLE_KERNEL_OMP=${{ matrix.enable_kernel_omp }}, ENABLE_KERNEL_AVX_STREAMING=${{ matrix.enable_kernel_avx_streaming }}), ENABLE_LAPACK=${{ matrix.enable_lapack }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5
@@ -119,13 +111,12 @@ jobs:
 
   cpptestswithOpenBLAS:
     if: ${{ !contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
-    needs: [determine_runner]
     strategy:
       matrix:
         pl_backend: ["lightning_qubit"]
     timeout-minutes: 60
     name: C++ Tests (${{ matrix.pl_backend }}, blas-ON)
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5
@@ -180,15 +171,14 @@ jobs:
 
   build_and_cache_Kokkos:
     name: "Build and cache Kokkos"
-    needs: [determine_runner]
     uses: ./.github/workflows/build_and_cache_Kokkos_linux.yml
     with:
-      runs_on: ${{ needs.determine_runner.outputs.runner_group }}
+      runs_on: pl-4-core-large-runner
       os: ubuntu-22.04
 
   cpptestswithKokkos:
     if: ${{ !contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
-    needs: [build_and_cache_Kokkos, determine_runner]
+    needs: [build_and_cache_Kokkos]
     strategy:
       matrix:
         os: [ubuntu-22.04]
@@ -197,7 +187,7 @@ jobs:
         kokkos_version: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.kokkos_version) }}
     timeout-minutes: 60
     name: C++ Tests (${{ matrix.pl_backend }}, kokkos-${{ matrix.kokkos_version }}, model-${{ matrix.exec_model }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5
@@ -288,15 +278,15 @@ jobs:
   cpptestsWithMultipleBackends:
   # Device-specific tests are performed for both. Device-agnostic tests default to LightningQubit.
     if: ${{ !contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
-    needs: [build_and_cache_Kokkos, determine_runner]
+    needs: [build_and_cache_Kokkos]
     strategy:
       matrix:
         os: [ubuntu-22.04]
         exec_model: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.exec_model) }}
         kokkos_version: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.kokkos_version) }}
     timeout-minutes: 60
     name: C++ Tests (multiple-backends, kokkos-${{ matrix.kokkos_version }}, model-${{ matrix.exec_model }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5

diff --git a/.github/workflows/tests_lkcpu_python.yml b/.github/workflows/tests_lkcpu_python.yml
@@ -43,14 +43,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-22.04
-
   build_and_cache_Kokkos:
     if: github.event.pull_request.draft == false
     name: "Build and cache Kokkos"
@@ -60,7 +52,7 @@ jobs:
       kokkos_version: "4.3.01"
 
   build_lightning_kokkos_wheels:
-    needs: [determine_runner, build_and_cache_Kokkos]
+    needs: [build_and_cache_Kokkos]
     strategy:
       matrix:
         os: [ubuntu-22.04]
@@ -69,7 +61,7 @@ jobs:
         kokkos_version: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.kokkos_version) }}
     timeout-minutes: 60
     name: Build (${{ matrix.pl_backend }}, kokkos-${{ matrix.kokkos_version }}, model-${{ matrix.exec_model }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - name: Checkout PennyLane-Lightning
@@ -149,7 +141,7 @@ jobs:
             ${{ github.workspace }}/wheel_${{ matrix.pl_backend }}-${{ matrix.exec_model }}.whl
 
   test_lightning_kokkos_wheels:
-    needs: [determine_runner, build_and_cache_Kokkos, build_lightning_kokkos_wheels]
+    needs: [build_and_cache_Kokkos, build_lightning_kokkos_wheels]
     strategy:
       matrix:
         os: [ubuntu-22.04]
@@ -159,7 +151,7 @@ jobs:
         kokkos_version: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.kokkos_version) }}
     timeout-minutes: 60
     name: Python Tests (${{ matrix.pl_backend }}, kokkos-${{ matrix.kokkos_version }}, model-${{ matrix.exec_model }}, test-group-${{ matrix.group }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5
@@ -284,9 +276,9 @@ jobs:
 
   upload-to-codecov-linux-python:
     if: github.event_name == 'pull_request'
-    needs: [determine_runner, test_lightning_kokkos_wheels]
+    needs: [test_lightning_kokkos_wheels]
     name: Upload python coverage data to codecov
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4

diff --git a/.github/workflows/tests_lqcpu_python.yml b/.github/workflows/tests_lqcpu_python.yml
@@ -33,22 +33,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-22.04
-
   build_lightning_qubit_wheels:
-    needs: [determine_runner]
     strategy:
       matrix:
         pl_backend: ["lightning_qubit"]
         blas: ["OFF", "ON"]
     timeout-minutes: 60
     name: Build (${{ matrix.pl_backend }}, blas-${{ matrix.blas }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - name: Make disk space
@@ -121,15 +113,15 @@ jobs:
             ${{ github.workspace }}/wheel_${{ matrix.pl_backend }}-${{ matrix.blas }}.whl
 
   test_lightning_qubit_wheels:
-    needs: [determine_runner, build_lightning_qubit_wheels]
+    needs: [build_lightning_qubit_wheels]
     strategy:
       matrix:
         pl_backend: ["lightning_qubit"]
         blas: ["OFF", "ON"]
         group: [1, 2, 3, 4]
     timeout-minutes: 60
     name: Python Tests (${{ matrix.pl_backend }}, blas-${{ matrix.blas }}, test-group-${{ matrix.group }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - uses: actions/setup-python@v5
@@ -232,9 +224,9 @@ jobs:
 
   upload-to-codecov-linux-python:
     if: github.event_name == 'pull_request'
-    needs: [determine_runner, test_lightning_qubit_wheels]
+    needs: [test_lightning_qubit_wheels]
     name: Upload python coverage data to codecov
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4

diff --git a/.github/workflows/tests_without_binary.yml b/.github/workflows/tests_without_binary.yml
@@ -30,17 +30,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-22.04
-
   pythontests:
-    needs: [determine_runner]
     timeout-minutes: 30
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
     strategy:
       matrix:
         pl_backend: ["lightning_qubit", "lightning_kokkos", "lightning_gpu", "lightning_tensor"]

diff --git a/.github/workflows/wheel_linux_aarch64.yml b/.github/workflows/wheel_linux_aarch64.yml
@@ -37,7 +37,7 @@ jobs:
     needs: [set_wheel_build_matrix]
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [pl-4-core-large-runner]
         arch: [aarch64]
         exec_model: ${{ fromJson(needs.set_wheel_build_matrix.outputs.exec_model) }}
         kokkos_version: ${{ fromJson(needs.set_wheel_build_matrix.outputs.kokkos_version) }}
@@ -94,7 +94,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [pl-4-core-large-runner]
         arch: [aarch64]
         pl_backend: ["lightning_kokkos", "lightning_qubit"]
         cibw_build: ${{ fromJson(needs.set_wheel_build_matrix.outputs.python_version) }}

diff --git a/.github/workflows/wheel_linux_aarch64_cuda.yml b/.github/workflows/wheel_linux_aarch64_cuda.yml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [pl-4-core-large-runner]
         arch: [aarch64]
         pl_backend: ["lightning_gpu", "lightning_tensor"]
         cuda_version: ["12"]

diff --git a/.github/workflows/wheel_linux_x86_64.yml b/.github/workflows/wheel_linux_x86_64.yml
@@ -36,23 +36,16 @@ jobs:
     with:
       event_name: ${{ github.event_name }}
 
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-latest
-
   build_dependencies:
-    needs: [set_wheel_build_matrix, determine_runner]
+    needs: [set_wheel_build_matrix]
     strategy:
       matrix:
         exec_model: ${{ fromJson(needs.set_wheel_build_matrix.outputs.exec_model) }}
         kokkos_version: ${{ fromJson(needs.set_wheel_build_matrix.outputs.kokkos_version) }}
         container_img: ["quay.io/pypa/manylinux_2_28_x86_64"]
     timeout-minutes: 30
     name: Kokkos core (${{ matrix.exec_model }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
     container: ${{ matrix.container_img }}
 
     steps:
@@ -100,7 +93,7 @@ jobs:
           cd -
 
   linux-wheels-x86-64:
-    needs: [set_wheel_build_matrix, build_dependencies, determine_runner]
+    needs: [set_wheel_build_matrix, build_dependencies]
     strategy:
       fail-fast: false
       matrix:
@@ -112,7 +105,7 @@ jobs:
         container_img: ["quay.io/pypa/manylinux_2_28_x86_64"]
     timeout-minutes: 30
     name: Linux::${{ matrix.arch }} - ${{ matrix.pl_backend }} (Python ${{ fromJson('{ "cp310-*":"3.10","cp311-*":"3.11","cp312-*":"3.12" }')[matrix.cibw_build] }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
     container:
       image: ${{ matrix.container_img }}
       options: --privileged

diff --git a/.github/workflows/wheel_linux_x86_64_cuda.yml b/.github/workflows/wheel_linux_x86_64_cuda.yml
@@ -32,16 +32,8 @@ jobs:
     with:
       event_name: ${{ github.event_name }}
 
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-latest
-
-
   linux-wheels-x86-64:
-    needs: [set_wheel_build_matrix, determine_runner]
+    needs: [set_wheel_build_matrix]
     strategy:
       fail-fast: false
       matrix:
@@ -52,7 +44,7 @@ jobs:
         container_img: ["quay.io/pypa/manylinux_2_28_x86_64"]
     timeout-minutes: 30
     name: Linux::${{ matrix.arch }} - ${{ matrix.pl_backend }} CUDA ${{ matrix.cuda_version }} (Python ${{ fromJson('{"cp310-*":"3.10","cp311-*":"3.11","cp312-*":"3.12" }')[matrix.cibw_build] }})
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
     container:
       image: ${{ matrix.container_img }}
       options: --privileged

diff --git a/.github/workflows/wheel_noarch.yml b/.github/workflows/wheel_noarch.yml
@@ -24,26 +24,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  determine_runner:
-    if: github.event.pull_request.draft == false
-    name: Determine runner type to use
-    uses: ./.github/workflows/determine-workflow-runner.yml
-    with:
-      default_runner: ubuntu-latest
-
-
   build-pure-python-wheel:
     if: |
       github.event_name != 'pull_request' ||
       contains(github.event.pull_request.labels.*.name, 'ci:build_wheels') ||
       github.event_name == 'workflow_dispatch'
-    needs: [determine_runner]
     strategy:
       matrix:
         pl_backend: ["lightning_gpu", "lightning_kokkos", "lightning_qubit"]
     timeout-minutes: 30
     name: Linux - Pure Python wheels - ${{ matrix.pl_backend }} (Python 3.10)
-    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    runs-on: pl-4-core-large-runner
 
     steps:
       - name: Checkout PennyLane-Lightning