Add new GPU workflow

.github/workflows/scripts/filter_git_diff.py

@@ -0,0 +1,39 @@
+# At the moment this script will only filter out the easystack files
+# Next step could be to filter out other subpaths 
+# by using the GITHUB action environment variables and argsparse.
+
+# Can also distiguish between modified, added, ... files
+# That information is in the string that is generated by git diff
+# For filtering easystacks this was however not necessary 
+# since we want to check both added and modified files
+
+# Todo: This script should eventually be its own github action. When it can be used for filtering more files. And all the todo have been addressed.
+
+import os
+
+diff = os.getenv('CHANGED')
+env_file = os.getenv('GITHUB_ENV')
+
+diff_list = diff.split('\n')
+
+diff_filter_path = 'easystacks'
+
+diff_filtered = ''
+
+for line in diff_list:
+    status = line.split('\t')[0]
+    file = line.split('\t')[1] 
+    if file.startswith(diff_filter_path):
+        # Ignoring the status assigned to the file
+        diff_filtered += file + ' '
+
+if diff_filtered != '':
+    # Todo: Can only handle 1 level paths now
+    # If we want to pass multi-level paths this will need to be updated
+    # Name of the env_var can change based on the file or path
+    env_var = 'CHANGED_' + diff_filter_path.upper()
+    set_var = env_var + "=" + diff_filtered
+
+    # This adds the environment variable to the github action environment
+    with open(env_file, 'a') as file:
+        file.write(set_var)

.github/workflows/scripts/parse_missing-installations-output.py

@@ -0,0 +1,23 @@
+import os
+import re
+
+missing = os.environ['missing']
+missing = missing.split('\n')
+missing_cuda = []
+missing_cpu = []
+for ec in missing:
+    if re.search('CUDA', ec):
+        missing_cuda.append(ec)
+    else:
+        missing_cpu.append(ec)
+if len(missing_cpu) != 0 and len(missing_cuda) != 0:
+    print(f'Please open a seperate pr for these dependencies: {missing_cpu}')
+    os.write(2, b'Error: CPU dependencies for CUDA build must be build in a seperate pr')
+    exit(1)
+elif len(missing_cuda) != 0:
+    # TODO: Make this set the accelorator label?
+    print(f'Have fun installing the following gpu builds: {missing_cuda}')
+elif len(missing_cpu) != 0:
+    print(f'Have fun installing the following gpu builds: {missing_cpu}')
+else:
+    print('no missing modules')

.github/workflows/test-check_easystacks.yml

@@ -0,0 +1,87 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Check new and changed easystacks
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+env:
+  EESSI_ACCELERATOR_TARGETS: |
+    x86_64/amd/zen2:
+      - nvidia/cc80
+    x86_64/amd/zen3:
+      - nvidia/cc80
+jobs:
+  check_missing:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2023.06
+        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/amd/zen4
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+            fetch-depth: 0
+        - name: Check for changed easystackfiles in pr
+          run: |
+            echo "GITHUB_BASE_REF = ${GITHUB_BASE_REF}"
+            echo $GITHUB_ENV
+            echo "git diff --no-renames --name-status ${GITHUB_BASE_REF}..HEAD"
+            export CHANGED=$(git diff --no-renames --name-status origin/${GITHUB_BASE_REF}..HEAD)
+            echo $CHANGED
+            python .github/workflows/scripts/filter_git_diff.py
+        - name: Mount EESSI CernVM-FS pilot repository
+          if: env.CHANGED_EASYSTACKS
+          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: software.eessi.io
+        - name: Test check_missing_installations.sh script
+          if: env.CHANGED_EASYSTACKS
+          run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
+              module load EasyBuild
+              which eb
+              eb --version
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              env | grep ^EESSI | sort
+
+              # check the changed easystack files
+              echo "check for missing installation in changed easystack files"
+              easystack_files=(${CHANGED_EASYSTACKS})
+
+              # check for missing installations if a file in the easystacks directory is changed
+              # check easystack file for the use of `--from-pr` 
+              for easystack_file in ${easystack_files[@]}; do
+                  echo "check for the use of from-pr in ${easystack_file}"
+                  if grep -q "from-pr" ${easystack_file}; then
+                      echo "ERROR: from-pr is found in ${easystack_file} please use from-commit" >&2
+                      exit 1
+                  fi
+
+                  echo "check missing installatios for ${easystack_file}..."
+                  ./check_missing_installations.sh ${easystack_file} || ec=$?
+                  export eb_missing_out=$(./check_missing_installations.sh ${easystack_file})
+                  if [[ ${ec} -ne 0 ]]; then 
+                      echo "missing installations found for ${easystack_file}!" >&2
+                      export missing=$(env | grep .eb\) | tr -d \*)
+                      python .github/workflows/scripts/parse_missing-installations-output.py 
+                  fi
+              done

.github/workflows/test-software.eessi.io.yml

@@ -94,8 +94,8 @@ jobs:
                   module load EasyBuild/${eb_version}
                   which eb
                   eb --version
-                  software-layer-scripts/check_missing_installations.sh ${easystack_file}
-                  ec=$?
+                  ec=0
+                  software-layer-scripts/check_missing_installations.sh ${easystack_file} || ec=$?
                   if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
               done
 
@@ -114,8 +114,8 @@ jobs:
                           module load EasyBuild/${eb_version}
                           which eb
                           eb --version
-                          software-layer-scripts/check_missing_installations.sh ${easystack_file}
-                          ec=$?
+                          ec=0
+                          software-layer-scripts/check_missing_installations.sh ${easystack_file} || ec=$?
                           if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
                       done
                       module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all