From 6f8662ec99152d6845afd88b0294badde8489481 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 12 Jun 2024 10:41:32 +0200 Subject: [PATCH 01/12] DEBUG only {2023.06,2023a} PyTorch-bundle v2.1.2 - PR to help debugging various issues when building PyTorch-bundle - includes a fix for `find_library` provided by `ctypes.util` which prevented importing `sndfile` - includes a fix for `aarch64/generic` where importing `sentencepiece` lead to the error `libtcmalloc_minimal.so.4: cannot allocate memory in static TLS block` - includes a fix for the extension `torchvision` where some library was not compiled with `jpeg` support, hence some tests failed --- EESSI-install-software.sh | 6 + .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 6 +- eb_hooks.py | 125 ++++++++++++++++++ scripts/extra/custom_ctypes-1.2.eb | 29 ++++ .../extra/eessi-2023.06-extra-packages.yml | 2 + scripts/extra/install_extra_packages.sh | 95 +++++++++++++ 6 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 scripts/extra/custom_ctypes-1.2.eb create mode 100644 scripts/extra/eessi-2023.06-extra-packages.yml create mode 100755 scripts/extra/install_extra_packages.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7d358e205a..423746ae03 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -249,6 +249,12 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi +# Install extra software that is needed (e.g., for providing a custom ctypes +# library when needed) +cd ${TOPDIR}/scripts/extra +./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +cd ${TOPDIR} + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 27c18a487e..8c1ab8b5b4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -18,7 +18,11 @@ easyconfigs: - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - WhatsHap-2.2-foss-2023a.eb - BLAST+-2.14.1-gompi-2023a.eb: - options: + options: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb + - PyTorch-bundle-2.1.2-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20382 + options: + from-pr: 20382 diff --git a/eb_hooks.py b/eb_hooks.py index 8b0a11b0ed..bbf5200c86 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -5,6 +5,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess +from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option @@ -214,6 +215,30 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") +def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): + """ + Add exts_filter to soundfile extension in exts_list + """ + if ec.name == 'librosa' and ec.version in ('0.10.1',): + ec_dict = ec.asdict() + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") + ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path + exts_list_new = [] + for item in ec_dict['exts_list']: + if item[0] == 'soundfile': + ext_dict = item[2] + ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0], + EXTS_FILTER_PYTHON_PACKAGES[1]) + exts_list_new.append((item[0], item[1], ext_dict)) + else: + exts_list_new.append(item) + ec['exts_list'] = exts_list_new + print_msg("New exts_list: '%s'", ec['exts_list']) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target for OpenBLAS < 0.3.23""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') @@ -253,6 +278,46 @@ def parse_hook_pybind11_replace_catch2(ec, eprefix): build_deps[idx] = (catch2_name, catch2_version) +def parse_hook_pytorch_bundle_torchvision_setenv(ec, eprefix): + """ + Set TORCHVISION_{INCLUDE,LIBRARY}, initially for non-CUDA version only + """ + if ec.name == 'PyTorch-bundle' and ec.version in ['2.1.2']: + if not hasattr(ec, 'versionsuffix') or (ec.versionsuffix and not 'CUDA' in ec.versionsuffix): + print_msg("parse_hook for PyTorch-bundle without CUDA: extslist '%s'", ec['exts_list']) + print_msg("New exts_list: '%s'", ec['exts_list']) + ec_dict = ec.asdict() + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + libpng_root = os.path.join(eessi_software_path, "software", "libpng", "1.6.39-GCCcore-12.3.0") + libpng_include = os.path.join(libpng_root, 'include') + libpng_lib = os.path.join(libpng_root, 'lib') + libjpeg_turbo_root = os.path.join(eessi_software_path, "software", "libjpeg-turbo", "2.1.5.1-GCCcore-12.3.0") + libjpeg_turbo_include = os.path.join(libjpeg_turbo_root, 'include') + libjpeg_turbo_lib = os.path.join(libjpeg_turbo_root, 'lib') + exts_list_new = [] + torchvision_include = 'export TORCHVISION_INCLUDE=%s:%s' % (libpng_include, libjpeg_turbo_include) + torchvision_library = 'export TORCHVISION_LIBRARY=%s:%s' % (libpng_lib, libjpeg_turbo_lib) + for item in ec_dict['exts_list']: + if item[0] != 'torchvision': + exts_list_new.append(item) + else: + ext_dict = item[2] + if 'preinstallopts' in ext_dict: + raise EasyBuildError("found value for 'preinstallopts' for extension 'torchvision'," + " but expected NONE") + else: + # add preinstallopts + ext_dict['preinstallopts'] = torchvision_include + ' && ' + torchvision_library + ' && ' + exts_list_new.append((item[0], item[1], ext_dict)) + ec['exts_list'] = exts_list_new + print_msg("New exts_list: '%s'", ec['exts_list']) + else: + print_msg("parse_hook for PyTorch-bundle for CUDA -> leaving preinstallopts unchanged") + else: + raise EasyBuildError("PyTorch-bundle-specific hook triggered for non-PyTorch-bundle easyconfig?!") + + def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): """ Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. @@ -266,6 +331,30 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") +def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix): + """ + Disable using TC_Malloc on 'aarch64/generic' + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ec.name == 'SentencePiece' and ec.version in ['0.2.0']: + if cpu_target == CPU_TARGET_AARCH64_GENERIC: + print_msg("parse_hook for SentencePiece: OLD '%s'", ec['components']) + new_components = [] + for item in ec['components']: + if item[2]['easyblock'] == 'CMakeMake': + new_item = item[2] + new_item['configopts'] = '-DSPM_ENABLE_TCMALLOC=OFF' + new_components.append((item[0], item[1], new_item)) + else: + new_components.append(item) + ec['components'] = new_components + print_msg("parse_hook for SentencePiece: NEW '%s'", ec['components']) + else: + print_msg("parse_hook for SentencePiece on %s -> leaving configopts unchanged", cpu_target) + else: + raise EasyBuildError("SentencePiece-specific hook triggered for non-SentencePiece easyconfig?!") + + def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': @@ -659,14 +748,46 @@ def inject_gpu_property(ec): return ec +def pre_module_hook(self, *args, **kwargs): + """Main pre-module-check hook: trigger custom functions based on software name.""" + if self.name in PRE_MODULE_HOOKS: + PRE_MODULE_HOOKS[self.name](self, *args, **kwargs) + + +def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): + """ + Add EBPYTHONPREFIXES to modluafooter + """ + if self.name == 'librosa' and self.version == '0.10.1': + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") + key = 'modluafooter' + values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] + print_msg("Adding '%s' to modluafooter", values[0]) + if not key in self.cfg: + self.cfg[key] = '\n'.join(values) + else: + new_value = self.cfg[key] + for value in values: + if not value in new_value: + new_value = '\n'.join([new_value, value]) + self.cfg[key] = new_value + print_msg("Full modluafooter is '%s'", self.cfg[key]) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'librosa': parse_hook_librosa_custom_ctypes, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, + 'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, + 'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, 'UCX': parse_hook_ucx_eprefix, } @@ -710,3 +831,7 @@ def inject_gpu_property(ec): POST_SANITYCHECK_HOOKS = { 'CUDA': post_sanitycheck_cuda, } + +PRE_MODULE_HOOKS = { + 'librosa': pre_module_hook_librosa_augment_modluafooter, +} diff --git a/scripts/extra/custom_ctypes-1.2.eb b/scripts/extra/custom_ctypes-1.2.eb new file mode 100644 index 0000000000..35be6dcc41 --- /dev/null +++ b/scripts/extra/custom_ctypes-1.2.eb @@ -0,0 +1,29 @@ +## +# This is a contribution from the NESSI project +# Homepage: https://documentation.sigma2.no +# +# Authors:: Thomas Roeblitz +# License:: GPL-2.0-only +# +## + +easyblock = 'Tarball' + +name = 'custom_ctypes' +version = '1.2' + +homepage = 'https://github.com/ComputeCanada/custom_ctypes' +description = """custum_ctypes is a small Python package to fix the discovery of libraries with Python's ctypes module. It changes the behavior of find_library to return absolute paths to shared objects rather than just the names.""" + +toolchain = SYSTEM + +source_urls = ['https://github.com/ComputeCanada/custom_ctypes/archive/refs/tags'] +sources = ['%(version)s.tar.gz'] +checksums = ['3b30ce633c6a329169f2b10ff24b8eaaeef3fa208a66cdacdb53c22f02a88d9b'] + +sanity_check_paths = { + 'files': ['README.md'], + 'dirs': ['lib'], +} + +moduleclass = 'lib' diff --git a/scripts/extra/eessi-2023.06-extra-packages.yml b/scripts/extra/eessi-2023.06-extra-packages.yml new file mode 100644 index 0000000000..22670ec7a3 --- /dev/null +++ b/scripts/extra/eessi-2023.06-extra-packages.yml @@ -0,0 +1,2 @@ +easyconfigs: + - custom_ctypes-1.2.eb diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh new file mode 100755 index 0000000000..ccd2890864 --- /dev/null +++ b/scripts/extra/install_extra_packages.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# This script can be used to install extra packages under ${EESSI_SOFTWARE_PATH} + +# some logging +echo ">>> Running ${BASH_SOURCE}" + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) +source "${TOPDIR}"/../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -e, --easystack EASYSTACKFILE Easystack file which specifies easyconfigs to be installed." + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation" +} + +# Initialize variables +TEMP_DIR= +EASYSTACK_FILE= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -e|--easystack) + if [ -n "$2" ]; then + EASYSTACK_FILE="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +if [[ -z ${EASYSTACK_FILE} ]]; then + show_help + fatal_error "Error: need to specify easystack file" +fi + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH +export NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} extra.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" +export WORKING_DIR=${tmpdir} + +# load EasyBuild +ml EasyBuild + +# load NESSI-extend/2023.06-easybuild +ml NESSI-extend/2023.06-easybuild + +eb --show-config + +eb --easystack ${EASYSTACK_FILE} --robot + +# clean up tmpdir +rm -rf "${tmpdir}" From 548e8eadd63e287d7feeb3b3ce8c0ca4aaa403ff Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 12 Jun 2024 11:09:15 +0200 Subject: [PATCH 02/12] initially disable all fixes --- EESSI-install-software.sh | 6 +++--- eb_hooks.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 423746ae03..195239b9eb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -251,9 +251,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -cd ${TOPDIR}/scripts/extra -./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml -cd ${TOPDIR} +#cd ${TOPDIR}/scripts/extra +#./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +#cd ${TOPDIR} # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/eb_hooks.py b/eb_hooks.py index bbf5200c86..413675a60f 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -782,12 +782,12 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, - 'librosa': parse_hook_librosa_custom_ctypes, + #'librosa': parse_hook_librosa_custom_ctypes, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, - 'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, + #'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, - 'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, + #'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, 'UCX': parse_hook_ucx_eprefix, } @@ -833,5 +833,5 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): } PRE_MODULE_HOOKS = { - 'librosa': pre_module_hook_librosa_augment_modluafooter, + #'librosa': pre_module_hook_librosa_augment_modluafooter, } From 2a76a7ac38857619fa80d024e382d834ccc249ad Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 15 Jun 2024 13:06:22 +0200 Subject: [PATCH 03/12] enabled fixes for failing sanitycheck in librosa --- EESSI-install-software.sh | 6 +++--- eb_hooks.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 195239b9eb..423746ae03 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -251,9 +251,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -#cd ${TOPDIR}/scripts/extra -#./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml -#cd ${TOPDIR} +cd ${TOPDIR}/scripts/extra +./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +cd ${TOPDIR} # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/eb_hooks.py b/eb_hooks.py index 413675a60f..84bd4fff84 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -782,7 +782,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, - #'librosa': parse_hook_librosa_custom_ctypes, + 'librosa': parse_hook_librosa_custom_ctypes, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, #'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, @@ -833,5 +833,5 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): } PRE_MODULE_HOOKS = { - #'librosa': pre_module_hook_librosa_augment_modluafooter, + 'librosa': pre_module_hook_librosa_augment_modluafooter, } From 4c995a9dcb0af03032a43ae71e3d99cbabf67f55 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 15 Jun 2024 19:58:07 +0200 Subject: [PATCH 04/12] enabling fix for SentencePiece on aarch64/generic --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 84bd4fff84..d2d773ba4e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -787,7 +787,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): 'pybind11': parse_hook_pybind11_replace_catch2, #'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, - #'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, + 'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, 'UCX': parse_hook_ucx_eprefix, } From 489615dfe5f286674d1ad714e8ecd0a53765d9f0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 15 Jun 2024 21:31:46 +0200 Subject: [PATCH 05/12] extend fix for SentencePiece to neoverse_{n1,v1} --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index d2d773ba4e..7b1e24642f 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -333,11 +333,11 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix): """ - Disable using TC_Malloc on 'aarch64/generic' + Disable using TC_Malloc on 'aarch64' CPUs """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'SentencePiece' and ec.version in ['0.2.0']: - if cpu_target == CPU_TARGET_AARCH64_GENERIC: + if cpu_target in [CPU_TARGET_AARCH64_GENERIC, CPU_TARGET_NEOVERSE_N1, CPU_TARGET_NEOVERSE_V1]: print_msg("parse_hook for SentencePiece: OLD '%s'", ec['components']) new_components = [] for item in ec['components']: From cf63c4cb715b3c3756e2b5a2d655aba543866d74 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 29 Jun 2024 22:52:40 +0200 Subject: [PATCH 06/12] move Python-bundle to easystack file for EB v4.9.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..18bf34b94f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - PyTorch-bundle-2.1.2-foss-2023a.eb From 822bcd685387b12a7a57fb8b64b7511e2a3131b6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 29 Jun 2024 22:53:09 +0200 Subject: [PATCH 07/12] move Python-bundle to easystack file for EB v4.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 8c1ab8b5b4..40169d47b5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,7 +22,3 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - PyTorch-bundle-2.1.2-foss-2023a.eb: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20382 - options: - from-pr: 20382 From b336e7f77a2c1d396c0ad87111ae347c86772a89 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 1 Aug 2024 09:07:33 +0200 Subject: [PATCH 08/12] disable initial fix for ctypes after general solution got adopted - PR #655 implements a general fix for the import error --- EESSI-install-software.sh | 6 +++--- eb_hooks.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 423746ae03..195239b9eb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -251,9 +251,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -cd ${TOPDIR}/scripts/extra -./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml -cd ${TOPDIR} +#cd ${TOPDIR}/scripts/extra +#./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +#cd ${TOPDIR} # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/eb_hooks.py b/eb_hooks.py index ebac2d5a54..901963d99c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -862,7 +862,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, - 'librosa': parse_hook_librosa_custom_ctypes, + #'librosa': parse_hook_librosa_custom_ctypes, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, @@ -916,5 +916,5 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): } PRE_MODULE_HOOKS = { - 'librosa': pre_module_hook_librosa_augment_modluafooter, + #'librosa': pre_module_hook_librosa_augment_modluafooter, } From 1b8b0e581ee51212f6b814cd564653c1f467a03b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 8 Aug 2024 12:18:28 +0200 Subject: [PATCH 09/12] removed custom fix for ctypes --- scripts/extra/custom_ctypes-1.2.eb | 29 ------ .../extra/eessi-2023.06-extra-packages.yml | 2 - scripts/extra/install_extra_packages.sh | 95 ------------------- 3 files changed, 126 deletions(-) delete mode 100644 scripts/extra/custom_ctypes-1.2.eb delete mode 100644 scripts/extra/eessi-2023.06-extra-packages.yml delete mode 100755 scripts/extra/install_extra_packages.sh diff --git a/scripts/extra/custom_ctypes-1.2.eb b/scripts/extra/custom_ctypes-1.2.eb deleted file mode 100644 index 35be6dcc41..0000000000 --- a/scripts/extra/custom_ctypes-1.2.eb +++ /dev/null @@ -1,29 +0,0 @@ -## -# This is a contribution from the NESSI project -# Homepage: https://documentation.sigma2.no -# -# Authors:: Thomas Roeblitz -# License:: GPL-2.0-only -# -## - -easyblock = 'Tarball' - -name = 'custom_ctypes' -version = '1.2' - -homepage = 'https://github.com/ComputeCanada/custom_ctypes' -description = """custum_ctypes is a small Python package to fix the discovery of libraries with Python's ctypes module. It changes the behavior of find_library to return absolute paths to shared objects rather than just the names.""" - -toolchain = SYSTEM - -source_urls = ['https://github.com/ComputeCanada/custom_ctypes/archive/refs/tags'] -sources = ['%(version)s.tar.gz'] -checksums = ['3b30ce633c6a329169f2b10ff24b8eaaeef3fa208a66cdacdb53c22f02a88d9b'] - -sanity_check_paths = { - 'files': ['README.md'], - 'dirs': ['lib'], -} - -moduleclass = 'lib' diff --git a/scripts/extra/eessi-2023.06-extra-packages.yml b/scripts/extra/eessi-2023.06-extra-packages.yml deleted file mode 100644 index 22670ec7a3..0000000000 --- a/scripts/extra/eessi-2023.06-extra-packages.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - custom_ctypes-1.2.eb diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh deleted file mode 100755 index ccd2890864..0000000000 --- a/scripts/extra/install_extra_packages.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to install extra packages under ${EESSI_SOFTWARE_PATH} - -# some logging -echo ">>> Running ${BASH_SOURCE}" - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) -source "${TOPDIR}"/../utils.sh - -# Function to display help message -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Options:" - echo " --help Display this help message" - echo " -e, --easystack EASYSTACKFILE Easystack file which specifies easyconfigs to be installed." - echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" - echo " storage during the installation" -} - -# Initialize variables -TEMP_DIR= -EASYSTACK_FILE= - -# Parse command-line options -while [[ $# -gt 0 ]]; do - case "$1" in - --help) - show_help - exit 0 - ;; - -e|--easystack) - if [ -n "$2" ]; then - EASYSTACK_FILE="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - -t|--temp-dir) - if [ -n "$2" ]; then - TEMP_DIR="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - *) - show_help - fatal_error "Error: Unknown option: $1" - ;; - esac -done - -if [[ -z ${EASYSTACK_FILE} ]]; then - show_help - fatal_error "Error: need to specify easystack file" -fi - -# Make sure NESSI is initialised -check_eessi_initialised - -# As an installation location just use $EESSI_SOFTWARE_PATH -export NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} - -# we need a directory we can use for temporary storage -if [[ -z "${TEMP_DIR}" ]]; then - tmpdir=$(mktemp -d) -else - mkdir -p ${TEMP_DIR} - tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} extra.XXX) - if [[ ! -d "$tmpdir" ]] ; then - fatal_error "Could not create directory ${tmpdir}" - fi -fi -echo "Created temporary directory '${tmpdir}'" -export WORKING_DIR=${tmpdir} - -# load EasyBuild -ml EasyBuild - -# load NESSI-extend/2023.06-easybuild -ml NESSI-extend/2023.06-easybuild - -eb --show-config - -eb --easystack ${EASYSTACK_FILE} --robot - -# clean up tmpdir -rm -rf "${tmpdir}" From 16eadc5d19859cd6174d3e6afc63259099594151 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 8 Aug 2024 12:19:50 +0200 Subject: [PATCH 10/12] added additional whitespaces --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 40169d47b5..27c18a487e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -18,7 +18,7 @@ easyconfigs: - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - WhatsHap-2.2-foss-2023a.eb - BLAST+-2.14.1-gompi-2023a.eb: - options: + options: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb From d622bdcd5c550f9df6998ae914d01a1f7d711bc8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 8 Aug 2024 12:20:54 +0200 Subject: [PATCH 11/12] no need to install extra packages anymore --- EESSI-install-software.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 195239b9eb..7d358e205a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -249,12 +249,6 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi -# Install extra software that is needed (e.g., for providing a custom ctypes -# library when needed) -#cd ${TOPDIR}/scripts/extra -#./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml -#cd ${TOPDIR} - # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then From e2078787a28226630e5a72877565b7b8878ae789 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 8 Aug 2024 12:23:41 +0200 Subject: [PATCH 12/12] remove obsolete hooks --- eb_hooks.py | 100 ---------------------------------------------------- 1 file changed, 100 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 901963d99c..5570e2052b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -6,7 +6,6 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess -from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option @@ -221,30 +220,6 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") -def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): - """ - Add exts_filter to soundfile extension in exts_list - """ - if ec.name == 'librosa' and ec.version in ('0.10.1',): - ec_dict = ec.asdict() - eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") - ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path - exts_list_new = [] - for item in ec_dict['exts_list']: - if item[0] == 'soundfile': - ext_dict = item[2] - ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0], - EXTS_FILTER_PYTHON_PACKAGES[1]) - exts_list_new.append((item[0], item[1], ext_dict)) - else: - exts_list_new.append(item) - ec['exts_list'] = exts_list_new - print_msg("New exts_list: '%s'", ec['exts_list']) - else: - raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") - - def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target for OpenBLAS < 0.3.23""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') @@ -284,46 +259,6 @@ def parse_hook_pybind11_replace_catch2(ec, eprefix): build_deps[idx] = (catch2_name, catch2_version) -def parse_hook_pytorch_bundle_torchvision_setenv(ec, eprefix): - """ - Set TORCHVISION_{INCLUDE,LIBRARY}, initially for non-CUDA version only - """ - if ec.name == 'PyTorch-bundle' and ec.version in ['2.1.2']: - if not hasattr(ec, 'versionsuffix') or (ec.versionsuffix and not 'CUDA' in ec.versionsuffix): - print_msg("parse_hook for PyTorch-bundle without CUDA: extslist '%s'", ec['exts_list']) - print_msg("New exts_list: '%s'", ec['exts_list']) - ec_dict = ec.asdict() - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - libpng_root = os.path.join(eessi_software_path, "software", "libpng", "1.6.39-GCCcore-12.3.0") - libpng_include = os.path.join(libpng_root, 'include') - libpng_lib = os.path.join(libpng_root, 'lib') - libjpeg_turbo_root = os.path.join(eessi_software_path, "software", "libjpeg-turbo", "2.1.5.1-GCCcore-12.3.0") - libjpeg_turbo_include = os.path.join(libjpeg_turbo_root, 'include') - libjpeg_turbo_lib = os.path.join(libjpeg_turbo_root, 'lib') - exts_list_new = [] - torchvision_include = 'export TORCHVISION_INCLUDE=%s:%s' % (libpng_include, libjpeg_turbo_include) - torchvision_library = 'export TORCHVISION_LIBRARY=%s:%s' % (libpng_lib, libjpeg_turbo_lib) - for item in ec_dict['exts_list']: - if item[0] != 'torchvision': - exts_list_new.append(item) - else: - ext_dict = item[2] - if 'preinstallopts' in ext_dict: - raise EasyBuildError("found value for 'preinstallopts' for extension 'torchvision'," - " but expected NONE") - else: - # add preinstallopts - ext_dict['preinstallopts'] = torchvision_include + ' && ' + torchvision_library + ' && ' - exts_list_new.append((item[0], item[1], ext_dict)) - ec['exts_list'] = exts_list_new - print_msg("New exts_list: '%s'", ec['exts_list']) - else: - print_msg("parse_hook for PyTorch-bundle for CUDA -> leaving preinstallopts unchanged") - else: - raise EasyBuildError("PyTorch-bundle-specific hook triggered for non-PyTorch-bundle easyconfig?!") - - def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): """ Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. @@ -828,45 +763,14 @@ def inject_gpu_property(ec): return ec -def pre_module_hook(self, *args, **kwargs): - """Main pre-module-check hook: trigger custom functions based on software name.""" - if self.name in PRE_MODULE_HOOKS: - PRE_MODULE_HOOKS[self.name](self, *args, **kwargs) - - -def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): - """ - Add EBPYTHONPREFIXES to modluafooter - """ - if self.name == 'librosa' and self.version == '0.10.1': - eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") - key = 'modluafooter' - values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] - print_msg("Adding '%s' to modluafooter", values[0]) - if not key in self.cfg: - self.cfg[key] = '\n'.join(values) - else: - new_value = self.cfg[key] - for value in values: - if not value in new_value: - new_value = '\n'.join([new_value, value]) - self.cfg[key] = new_value - print_msg("Full modluafooter is '%s'", self.cfg[key]) - else: - raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") - - PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, - #'librosa': parse_hook_librosa_custom_ctypes, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, - #'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, 'UCX': parse_hook_ucx_eprefix, @@ -914,7 +818,3 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): POST_SANITYCHECK_HOOKS = { 'CUDA': post_sanitycheck_cuda, } - -PRE_MODULE_HOOKS = { - #'librosa': pre_module_hook_librosa_augment_modluafooter, -}