Skip to content

add hook that copies the easybuild subdirectory of every installation to a central reprod directory #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Jul 15, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion bot/check-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,9 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
# extract directories/entries from tarball content
modules_entries=$(grep "${prefix}/modules" ${tmpfile})
software_entries=$(grep "${prefix}/software" ${tmpfile})
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
reprod_entries=$(grep "${prefix}/reprod" ${tmpfile})
reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod")
other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
Expand Down Expand Up @@ -531,6 +533,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
fi
comment_artifacts_list="${comment_artifacts_list}</pre>"
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)"
comment_artifacts_list="${comment_artifacts_list}<pre>"
if [[ ! -z ${reprod_shortened} ]]; then
while IFS= read -r reprod ; do
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${reprod})"
done <<< "${reprod_shortened}"
else
comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
fi
comment_artifacts_list="${comment_artifacts_list}</pre>"
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
comment_artifacts_list="${comment_artifacts_list}<pre>"
if [[ ! -z ${other_shortened} ]]; then
Expand Down
5 changes: 5 additions & 0 deletions create_tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,13 @@ for subdir in ${sw_subdirs}; do
for package_version in $(cat ${module_files_list}); do
echo "handling ${package_version}"
find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
# if there is a directory for this installation in the stack's reprod directory, include that too
if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then
find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
fi
done
fi

done

# add a bit debug output
Expand Down
43 changes: 41 additions & 2 deletions eb_hooks.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
# Hooks to customize how EasyBuild installs software in EESSI
# see https://docs.easybuild.io/en/latest/Hooks.html
import datetime
import glob
import os
import re

import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import obtain_config_guess
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.tools import config
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option, update_build_option
from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which
from easybuild.tools.config import build_option, install_path, update_build_option
from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
Expand Down Expand Up @@ -46,6 +48,9 @@
# Make sure a single environment variable name is used for this throughout the hooks
EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220"

STACK_REPROD_SUBDIR = 'reprod'


def is_gcccore_1220_based(**kwargs):
# ecname, ecversion, tcname, tcversion):
"""
Expand Down Expand Up @@ -516,6 +521,20 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs):
del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR]


def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs):
"""
Post easyblock hook that copies the easybuild subdirectory of every installed application
to a central and timestamped location in the root of the software stack, e.g.:
/path/to/stack/reprod/MyApp/1.2-foss-2025a/20250102T12:34:56Z
"""

stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR)
now_utc_timestamp = datetime.datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S%Z')
app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg))
app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild')
copy_dir(app_easybuild_dir, app_reprod_dir)


# Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set
# so that the modules can be succesfully loaded without printing the error (so that we can create a module
# _with_ the warning for the current software being installed)
Expand Down Expand Up @@ -1297,6 +1316,24 @@ def post_module_hook(self, *args, **kwargs):
post_module_hook_zen4_gcccore1220(self, *args, **kwargs)


# The post_easyblock_hook was introduced in EasyBuild 5.1.1.
# Older versions would fail if the function is defined anyway, as EasyBuild performs some checks on function names in hooks files.
if EASYBUILD_VERSION >= '5.1.1':
def post_easyblock_hook(self, *args, **kwargs):
"""Main post easyblock hook: trigger custom functions based on software name."""
if self.name in POST_EASYBLOCK_HOOKS:
POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs)

# Always trigger this one for EESSI CVMFS/site installations and version 2025.06 or newer, regardless of self.name
if os.getenv('EESSI_CVMFS_INSTALL') or os.getenv('EESSI_SITE_INSTALL'):
if os.getenv('EESSI_VERSION') and LooseVersion(os.getenv('EESSI_VERSION')) >= '2025.06':
post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs)
else:
self.log.debug("No CVMFS/site installation requested, not running post_easyblock_hook_copy_easybuild_subdir.")
else:
print_msg(f"Not enabling the post_easybuild_hook, as it requires EasyBuild 5.1.1 or newer.")


PARSE_HOOKS = {
'casacore': parse_hook_casacore_disable_vectorize,
'CGAL': parse_hook_cgal_toolchainopts_precise,
Expand Down Expand Up @@ -1365,6 +1402,8 @@ def post_module_hook(self, *args, **kwargs):

POST_MODULE_HOOKS = {}

POST_EASYBLOCK_HOOKS = {}

# Define parallelism limit operations
def divide_by_factor(parallel, factor):
"""Divide parallelism by given factor"""
Expand Down